diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv index 47e09bbc24ec03d77adc17a49613b12c1a5c21f2..0e5d623c2b0052a063456b4268230d4f203f80ea 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/mem_samples.csv @@ -22,3 +22,211 @@ 5, 1 6, 1 7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 4 +1, 629 +2, 629 +3, 165 +4, 629 +5, 629 +6, 629 +7, 629 +0, 909 +1, 905 +2, 935 +3, 915 +4, 911 +5, 917 +6, 921 +7, 915 +0, 1803 +1, 1803 +2, 1803 +3, 1803 +4, 1803 +5, 1803 +6, 1803 +7, 1803 +0, 43721 +1, 43157 +2, 43201 +3, 43883 +4, 43859 +5, 43865 +6, 43899 +7, 43859 +0, 44151 +1, 43583 +2, 43581 +3, 44281 +4, 44281 +5, 44281 +6, 44281 +7, 44281 +0, 47221 +1, 46653 +2, 46651 +3, 47351 +4, 47351 +5, 47351 +6, 47351 +7, 47351 +0, 47501 +1, 46933 +2, 46931 +3, 47631 +4, 47631 +5, 47631 +6, 47631 +7, 47631 +0, 47833 +1, 47289 +2, 47263 +3, 47969 +4, 47961 +5, 48023 +6, 47977 +7, 47989 +0, 48535 +1, 47967 +2, 47965 +3, 48665 +4, 48665 +5, 48665 +6, 48665 +7, 48665 +0, 48983 +1, 48415 +2, 48413 +3, 49113 +4, 49113 +5, 49113 +6, 49113 +7, 49113 +0, 51027 +1, 50457 +2, 50457 +3, 51157 +4, 51157 +5, 51157 +6, 51157 +7, 51157 +0, 51027 +1, 50457 +2, 50457 +3, 51157 +4, 51157 +5, 51157 +6, 51157 +7, 51157 +0, 82405 +1, 82427 +2, 82433 +3, 82431 +4, 82405 +5, 82409 +6, 82403 +7, 82369 +0, 82843 +1, 82851 +2, 82851 +3, 82849 +4, 82849 +5, 82849 +6, 82849 +7, 82849 +0, 84027 +1, 84035 +2, 84035 +3, 84033 +4, 84033 +5, 84033 +6, 84033 +7, 84033 +0, 84027 +1, 84035 +2, 84035 +3, 84033 +4, 84033 +5, 84033 +6, 84033 +7, 84033 +0, 84027 +1, 84035 +2, 84035 +3, 84033 +4, 84033 +5, 84033 +6, 84033 +7, 84033 +0, 85139 +1, 85145 +2, 85145 +3, 85143 +4, 85143 +5, 85143 +6, 85143 +7, 85143 +0, 87187 +1, 87193 +2, 87193 +3, 87191 +4, 87191 +5, 87191 +6, 87191 +7, 87191 +0, 87859 +1, 87889 +2, 87895 +3, 87877 +4, 87893 +5, 87867 +6, 87877 +7, 87871 +0, 108189 +1, 108267 +2, 108287 +3, 108273 +4, 108273 +5, 108273 +6, 108273 +7, 108273 +0, 108189 +1, 108267 +2, 108287 +3, 108273 +4, 108273 +5, 108273 +6, 108273 +7, 108273 +0, 2382 +1, 2382 +2, 2382 +3, 2382 +4, 2382 +5, 2638 +6, 107379 +7, 3662 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log index f71200f38f5a489e25fe046306751475fc60a07a..dcc3cdb8bbdbee801837e24e7759f54f588491f7 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1/train.log @@ -11,3 +11,6967 @@ W0826 14:56:38.411000 40721 torch/distributed/run.py:766] W0826 14:56:38.411000 40721 torch/distributed/run.py:766] ***************************************** W0826 14:56:38.411000 40721 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0826 14:56:38.411000 40721 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... True + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 10 + encoder_seq_length .............................. 4096 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 2560 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 3584 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.01 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 2560 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 2.0 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ None + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 20480 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 28 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 10 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... False + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 4096 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 05 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank5]:[W826 14:56:56.903435194 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 14:56:56.903444324 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 14:56:56.904460435 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 14:56:56.907063900 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 14:56:56.907120148 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 14:56:56.907162534 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 14:56:56.907230640 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.037 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 14:56:57.071097034 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.807 seconds +time to initialize megatron (seconds): 7.168 +[after megatron is initialized] datetime: 2026-08-26 14:57:02 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1352: UserWarning: Using a large number of experts (e.g. >=32) without fp32 routing. Consider enabling moe_router_dtype for better numerical stability. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 5060614656 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 22 +Params for bucket 1 (180289536 elements, 180289536 padded size): + module.output_layer.weight +Params for bucket 2 (73403904 elements, 73403904 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (146800640 elements, 146800640 padded size): + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight +Params for bucket 4 (103001600 elements, 103001600 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (146800640 elements, 146800640 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.bias + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.router.weight +Params for bucket 7 (146800640 elements, 146800640 padded size): + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight +Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.router.weight +Params for bucket 9 (146800640 elements, 146800640 padded size): + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 10 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.bias + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 11 (146800640 elements, 146800640 padded size): + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight +Params for bucket 12 (103001600 elements, 103001600 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.self_attention.linear_qkv.bias + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight +Params for bucket 13 (146800640 elements, 146800640 padded size): + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight +Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.self_attention.linear_qkv.bias +Params for bucket 15 (146800640 elements, 146800640 padded size): + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight +Params for bucket 16 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 17 (146800640 elements, 146800640 padded size): + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (103001600 elements, 103001600 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_qkv.bias + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight +Params for bucket 19 (146800640 elements, 146800640 padded size): + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight +Params for bucket 20 (103001600 elements, 103001600 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.bias + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 21 (146800640 elements, 146800640 padded size): + module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight +Params for bucket 22 (209890816 elements, 209890816 padded size): + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.bias + module.decoder.layers.0.self_attention.linear_proj.weight + module.embedding.word_embeddings.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 +Params for bucket 1 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 3 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 4 (55050240 elements, 55050240 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 +Params for bucket 5 (45875200 elements, 45875200 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 6 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 7 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 8 (55050240 elements, 55050240 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 +Params for bucket 9 (45875200 elements, 45875200 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 10 (45875200 elements, 45875200 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 +Params for bucket 11 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 12 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 13 (55050240 elements, 55050240 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 14 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 +Params for bucket 15 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 16 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 +Params for bucket 17 (55050240 elements, 55050240 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 18 (45875200 elements, 45875200 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 19 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 +Params for bucket 20 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 +Params for bucket 21 (55050240 elements, 55050240 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 +Params for bucket 22 (45875200 elements, 45875200 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (45875200 elements, 45875200 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 +Params for bucket 24 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 +Params for bucket 25 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 26 (55050240 elements, 55050240 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 27 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 +Params for bucket 28 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (55050240 elements, 55050240 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 31 (45875200 elements, 45875200 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 34 (55050240 elements, 55050240 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 +Params for bucket 36 (45875200 elements, 45875200 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 +Params for bucket 37 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 +Params for bucket 38 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 39 (55050240 elements, 55050240 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 40 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 +Params for bucket 41 (45875200 elements, 45875200 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 42 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 +Params for bucket 43 (55050240 elements, 55050240 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 +Params for bucket 44 (18350080 elements, 18350080 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 14:57:03 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 14:57:03 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (344.83, 351.07) + train/valid/test-data-iterators-setup ..........: (62.24, 71.92) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 14:57:03 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[553, 506, 524, 512, 524, 522, 506, 509, 528, 522, 518, 526, 495, 533, 482, 537, 527, 459, 555, 505, 514, 511, 500, 502, 512, 493, 514, 497, 498, 511, 566, 479, 518, 539, 488, 527, 501, 522, 512, 537, 510, 497, 498, 528, 515, 501, 481, 510, 512, 532, 525, 488, 505, 535, 494, 505, 493, 495, 498, 507, 531, 522, 514, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[543, 521, 531, 478, 515, 538, 554, 552, 524, 528, 521, 466, 522, 485, 507, 515, 525, 495, 514, 493, 488, 482, 456, 484, 493, 539, 524, 496, 537, 519, 555, 502, 488, 513, 515, 504, 509, 501, 495, 486, 512, 494, 550, 503, 485, 529, 519, 530, 499, 514, 507, 491, 508, 495, 531, 532, 508, 533, 529, 523, 527, 468, 558, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[556, 493, 548, 520, 503, 532, 515, 549, 507, 541, 527, 509, 550, 484, 548, 529, 511, 534, 467, 533, 486, 505, 535, 518, 504, 517, 510, 498, 541, 507, 550, 515, 514, 462, 488, 502, 475, 482, 542, 475, 527, 472, 519, 502, 499, 493, 513, 503, 524, 498, 495, 513, 497, 499, 557, 479, 484, 557, 519, 496, 504, 505, 528, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[549, 511, 496, 506, 482, 538, 491, 479, 530, 497, 522, 531, 481, 522, 527, 520, 485, 504, 529, 492, 514, 499, 488, 494, 519, 480, 491, 553, 548, 502, 522, 503, 539, 509, 506, 490, 492, 538, 499, 500, 532, 558, 501, 504, 488, 530, 530, 503, 538, 540, 543, 480, 525, 519, 541, 558, 491, 516, 499, 473, 519, 497, 505, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[495, 520, 504, 525, 495, 487, 512, 531, 531, 533, 520, 530, 523, 501, 489, 483, 548, 541, 542, 481, 515, 528, 518, 533, 508, 526, 513, 530, 501, 535, 513, 506, 488, 531, 490, 542, 536, 541, 487, 509, 537, 519, 524, 527, 476, 494, 512, 526, 534, 480, 504, 516, 520, 512, 490, 504, 504, 493, 475, 491, 482, 532, 474, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[560, 501, 503, 528, 505, 533, 510, 507, 535, 529, 527, 527, 529, 490, 513, 533, 524, 525, 489, 500, 545, 529, 509, 498, 497, 533, 526, 511, 522, 509, 513, 484, 521, 490, 541, 474, 473, 482, 544, 510, 508, 467, 485, 505, 519, 461, 517, 524, 568, 479, 507, 532, 520, 517, 494, 530, 484, 503, 501, 558, 528, 498, 494, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[534, 518, 543, 527, 546, 506, 491, 473, 490, 505, 511, 528, 509, 520, 502, 473, 498, 510, 515, 520, 518, 569, 552, 493, 521, 477, 506, 538, 500, 495, 455, 523, 543, 497, 517, 521, 484, 536, 492, 515, 511, 506, 510, 484, 487, 523, 506, 482, 508, 551, 489, 498, 518, 527, 536, 541, 556, 514, 515, 487, 517, 495, 515, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[494, 495, 514, 534, 512, 529, 517, 513, 505, 508, 498, 473, 479, 514, 529, 505, 493, 537, 514, 488, 547, 499, 515, 539, 538, 533, 537, 507, 520, 520, 534, 543, 485, 520, 505, 503, 502, 464, 564, 511, 523, 567, 504, 476, 524, 513, 509, 536, 506, 499, 513, 480, 516, 495, 514, 507, 486, 526, 513, 502, 522, 525, 469, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 506, 524, 512, 524, 522, 506, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 509, 518, 526, 495, 533, 482, 537, 0, 0, 0, 0, 0, 0, 0, 0, 527, 459, 555, 505, 514, 511, 500, 502, 4, 0, 0, 0, 0, 0, 0, 0, 512, 493, 514, 497, 498, 511, 539, 479, 0, 0, 0, 0, 0, 0, 0, 0, 518, 539, 488, 527, 501, 522, 512, 537, 27, 7, 2, 0, 0, 0, 0, 0, 510, 497, 498, 528, 515, 501, 481, 510, 11, 9, 0, 0, 0, 0, 0, 0, 499, 532, 525, 488, 505, 535, 494, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 495, 498, 507, 531, 522, 514, 488, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4128, 4077, 4043, 4180, 4060, 4083, 4085] avg=4096.0000 max=4180 max_rank=4 min=4043 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[505, 511, 496, 506, 482, 538, 491, 479, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 522, 531, 481, 522, 527, 520, 0, 0, 0, 0, 0, 0, 0, 0, 485, 504, 529, 492, 514, 499, 488, 494, 3, 0, 0, 0, 0, 0, 0, 0, 519, 480, 491, 553, 548, 502, 498, 503, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 506, 490, 492, 538, 499, 500, 24, 7, 2, 0, 0, 0, 0, 0, 532, 558, 501, 504, 488, 530, 530, 503, 12, 9, 0, 0, 0, 0, 0, 0, 524, 540, 543, 480, 525, 519, 541, 558, 0, 0, 0, 0, 0, 0, 0, 0, 491, 516, 499, 473, 519, 497, 505, 500, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4118, 4008, 4094, 4106, 4167, 4230, 4037] avg=4096.0000 max=4230 max_rank=6 min=4008 min_rank=0 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16540 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[500, 521, 531, 478, 515, 538, 554, 552, 0, 0, 0, 0, 0, 0, 0, 0, 524, 515, 521, 466, 522, 485, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 514, 493, 488, 482, 456, 484, 4, 0, 0, 0, 0, 0, 0, 0, 493, 539, 524, 496, 537, 519, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 488, 513, 515, 504, 509, 501, 495, 486, 26, 7, 2, 0, 0, 0, 0, 0, 512, 494, 550, 503, 485, 529, 519, 530, 11, 9, 0, 0, 0, 0, 0, 0, 486, 514, 507, 491, 508, 495, 531, 532, 0, 0, 0, 0, 0, 0, 0, 0, 508, 533, 529, 523, 527, 468, 558, 510, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 4055, 3941, 4139, 4046, 4142, 4064, 4192] avg=4096.0000 max=4192 max_rank=7 min=3941 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[512, 493, 548, 520, 503, 532, 515, 549, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 527, 509, 550, 484, 548, 529, 0, 0, 0, 0, 0, 0, 0, 0, 511, 534, 467, 533, 486, 505, 535, 518, 4, 0, 0, 0, 0, 0, 0, 0, 504, 517, 510, 498, 541, 507, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 514, 462, 488, 502, 475, 482, 542, 475, 26, 7, 2, 0, 0, 0, 0, 0, 527, 472, 519, 502, 499, 493, 513, 503, 11, 9, 0, 0, 0, 0, 0, 0, 511, 498, 495, 513, 497, 499, 557, 479, 0, 0, 0, 0, 0, 0, 0, 0, 484, 557, 519, 496, 504, 505, 528, 503, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4182, 4093, 4116, 3975, 4048, 4049, 4133] avg=4096.0000 max=4182 max_rank=1 min=3975 min_rank=4 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16205 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[456, 520, 504, 525, 495, 487, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 520, 520, 530, 523, 501, 489, 483, 0, 0, 0, 0, 0, 0, 0, 0, 548, 541, 542, 481, 515, 528, 518, 533, 4, 0, 0, 0, 0, 0, 0, 0, 508, 526, 513, 530, 501, 535, 489, 506, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 490, 542, 536, 541, 487, 509, 24, 6, 2, 0, 0, 0, 0, 0, 537, 519, 524, 527, 476, 494, 512, 526, 11, 9, 0, 0, 0, 0, 0, 0, 521, 480, 504, 516, 520, 512, 490, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 493, 475, 491, 482, 532, 474, 501, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4097, 4210, 4108, 4156, 4135, 4047, 3985] avg=4096.0000 max=4210 max_rank=2 min=3985 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[515, 501, 503, 528, 505, 533, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 527, 527, 529, 490, 513, 533, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 489, 500, 545, 529, 509, 498, 4, 0, 0, 0, 0, 0, 0, 0, 497, 533, 526, 511, 522, 509, 489, 484, 0, 0, 0, 0, 0, 0, 0, 0, 521, 490, 541, 474, 473, 482, 544, 510, 24, 7, 2, 0, 0, 0, 0, 0, 508, 467, 485, 505, 519, 461, 517, 524, 12, 9, 0, 0, 0, 0, 0, 0, 554, 479, 507, 532, 520, 517, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 484, 503, 501, 558, 528, 498, 494, 490, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4170, 4123, 4071, 4068, 4007, 4133, 4094] avg=4096.0000 max=4170 max_rank=1 min=4007 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[455, 495, 514, 534, 512, 529, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 505, 496, 498, 473, 479, 514, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0, 493, 537, 514, 488, 547, 499, 515, 539, 3, 0, 0, 0, 0, 0, 0, 0, 538, 533, 537, 507, 520, 520, 509, 543, 0, 0, 0, 0, 0, 0, 0, 0, 485, 520, 505, 503, 502, 464, 564, 511, 25, 6, 2, 0, 0, 0, 0, 0, 523, 567, 504, 476, 524, 513, 509, 536, 11, 9, 0, 0, 0, 0, 0, 0, 493, 499, 513, 480, 516, 495, 514, 507, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 513, 502, 522, 525, 469, 506, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 3999, 4135, 4207, 4087, 4172, 4017, 4082] avg=4096.0000 max=4207 max_rank=3 min=3999 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[482, 518, 543, 527, 546, 506, 491, 473, 0, 0, 0, 0, 0, 0, 0, 0, 490, 486, 511, 528, 509, 520, 502, 473, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 515, 520, 518, 569, 552, 493, 6, 0, 0, 0, 0, 0, 0, 0, 521, 477, 506, 538, 500, 495, 430, 523, 0, 0, 0, 0, 0, 0, 0, 0, 543, 497, 517, 521, 484, 536, 492, 515, 25, 12, 8, 0, 0, 0, 0, 0, 511, 506, 510, 484, 487, 523, 506, 482, 15, 13, 0, 0, 0, 0, 0, 0, 485, 551, 489, 498, 518, 527, 536, 541, 0, 0, 0, 0, 0, 0, 0, 0, 556, 514, 515, 487, 517, 495, 515, 521, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4019, 4181, 3990, 4150, 4037, 4145, 4160] avg=4096.0000 max=4181 max_rank=2 min=3990 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[499, 529, 546, 541, 516, 503, 478, 536, 512, 543, 533, 503, 507, 513, 483, 530, 496, 532, 491, 538, 510, 498, 520, 518, 539, 531, 561, 484, 529, 504, 502, 533, 489, 527, 532, 490, 489, 504, 497, 503, 506, 531, 512, 528, 513, 487, 506, 538, 478, 476, 505, 529, 473, 473, 549, 524, 486, 537, 522, 514, 478, 503, 506, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[529, 522, 547, 516, 528, 471, 517, 507, 513, 491, 509, 487, 536, 507, 500, 506, 520, 528, 525, 505, 493, 533, 535, 513, 526, 528, 533, 513, 527, 560, 498, 505, 520, 525, 484, 470, 503, 494, 515, 499, 505, 518, 488, 557, 491, 518, 527, 522, 511, 519, 465, 491, 480, 505, 516, 515, 513, 510, 500, 530, 528, 526, 464, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[522, 544, 512, 489, 504, 506, 510, 534, 503, 517, 541, 543, 552, 519, 531, 470, 512, 518, 515, 515, 530, 491, 487, 491, 482, 526, 524, 525, 498, 509, 508, 487, 496, 525, 539, 516, 488, 524, 492, 524, 499, 504, 523, 527, 474, 496, 522, 515, 537, 475, 493, 498, 470, 544, 496, 559, 512, 521, 553, 509, 520, 508, 504, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[501, 535, 501, 496, 480, 497, 498, 505, 538, 524, 504, 527, 527, 533, 550, 552, 467, 523, 497, 516, 529, 494, 541, 468, 498, 516, 514, 524, 532, 508, 485, 526, 561, 483, 524, 506, 484, 491, 510, 487, 541, 526, 515, 501, 474, 490, 505, 491, 559, 510, 548, 522, 496, 497, 506, 514, 503, 535, 530, 505, 527, 518, 500, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[514, 524, 531, 469, 500, 546, 531, 489, 493, 530, 528, 509, 494, 503, 508, 507, 545, 527, 542, 529, 536, 492, 516, 505, 538, 495, 488, 507, 507, 529, 519, 539, 515, 475, 517, 501, 514, 483, 537, 506, 514, 481, 498, 527, 506, 493, 483, 524, 546, 504, 492, 506, 522, 523, 481, 500, 508, 529, 528, 543, 523, 491, 450, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[530, 475, 514, 498, 517, 444, 533, 543, 533, 495, 526, 517, 476, 510, 509, 509, 497, 490, 523, 499, 512, 487, 540, 570, 495, 513, 507, 512, 543, 517, 525, 532, 547, 499, 548, 538, 528, 514, 519, 529, 548, 504, 501, 487, 513, 492, 493, 551, 502, 495, 503, 499, 536, 524, 483, 510, 537, 475, 534, 537, 519, 473, 470, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[488, 535, 541, 507, 548, 507, 561, 527, 524, 481, 491, 509, 535, 489, 534, 492, 540, 526, 487, 523, 501, 503, 493, 504, 504, 515, 551, 483, 515, 529, 500, 513, 489, 464, 543, 537, 538, 490, 503, 523, 515, 465, 545, 506, 528, 525, 557, 465, 495, 469, 473, 552, 536, 526, 529, 474, 550, 505, 472, 501, 507, 494, 524, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[498, 496, 539, 524, 527, 502, 553, 506, 532, 487, 519, 462, 517, 521, 545, 558, 527, 516, 516, 509, 518, 543, 545, 516, 490, 502, 527, 531, 522, 492, 527, 523, 519, 493, 516, 486, 445, 493, 496, 471, 535, 495, 474, 565, 503, 528, 505, 500, 507, 492, 503, 515, 535, 535, 540, 520, 464, 522, 497, 495, 494, 510, 507, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 529, 525, 541, 516, 503, 478, 536, 0, 0, 0, 0, 0, 0, 0, 0, 512, 543, 533, 503, 507, 513, 463, 530, 0, 0, 0, 0, 0, 0, 0, 0, 496, 532, 491, 538, 510, 498, 501, 518, 0, 0, 0, 0, 0, 0, 0, 0, 539, 531, 527, 484, 529, 504, 502, 533, 0, 0, 0, 0, 0, 0, 0, 0, 489, 527, 532, 490, 489, 504, 497, 503, 34, 5, 0, 0, 0, 0, 0, 0, 506, 531, 512, 528, 513, 487, 506, 538, 11, 0, 0, 0, 0, 0, 0, 0, 478, 476, 505, 529, 473, 473, 549, 524, 15, 9, 0, 0, 0, 0, 0, 0, 486, 537, 522, 514, 478, 503, 506, 505, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4104, 4084, 4149, 4070, 4132, 4031, 4071] avg=4096.0000 max=4149 max_rank=3 min=4031 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 524, 511, 469, 500, 546, 531, 489, 0, 0, 0, 0, 0, 0, 0, 0, 493, 530, 528, 509, 494, 503, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 545, 527, 542, 529, 536, 492, 497, 505, 0, 0, 0, 0, 0, 0, 0, 0, 538, 495, 459, 507, 507, 529, 519, 539, 0, 0, 0, 0, 0, 0, 0, 0, 515, 475, 517, 501, 514, 483, 537, 506, 29, 5, 0, 0, 0, 0, 0, 0, 514, 481, 498, 527, 506, 493, 483, 524, 11, 0, 0, 0, 0, 0, 0, 0, 546, 504, 492, 506, 522, 523, 481, 500, 16, 9, 0, 0, 0, 0, 0, 0, 508, 529, 528, 543, 523, 491, 450, 558, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4051, 4173, 4093, 4082, 4037, 4099, 4149] avg=4096.0000 max=4173 max_rank=2 min=4037 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 544, 493, 489, 504, 506, 510, 534, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 541, 543, 552, 519, 509, 470, 0, 0, 0, 0, 0, 0, 0, 0, 512, 518, 515, 515, 530, 491, 470, 491, 0, 0, 0, 0, 0, 0, 0, 0, 482, 526, 492, 525, 498, 509, 508, 487, 0, 0, 0, 0, 0, 0, 0, 0, 496, 525, 539, 516, 488, 524, 492, 524, 32, 5, 0, 0, 0, 0, 0, 0, 499, 504, 523, 527, 474, 496, 522, 515, 10, 0, 0, 0, 0, 0, 0, 0, 537, 475, 493, 498, 470, 544, 496, 559, 17, 8, 0, 0, 0, 0, 0, 0, 512, 521, 553, 509, 520, 508, 504, 490, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4154, 4042, 4027, 4141, 4070, 4097, 4135] avg=4096.0000 max=4154 max_rank=1 min=4027 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[488, 535, 520, 507, 548, 507, 561, 527, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 491, 509, 535, 489, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 540, 526, 487, 523, 501, 503, 476, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 518, 483, 515, 529, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 489, 464, 543, 537, 538, 490, 503, 523, 33, 5, 0, 0, 0, 0, 0, 0, 515, 465, 545, 506, 528, 525, 557, 465, 10, 0, 0, 0, 0, 0, 0, 0, 495, 469, 473, 552, 536, 526, 529, 474, 17, 9, 0, 0, 0, 0, 0, 0, 550, 505, 472, 501, 507, 494, 524, 512, 12, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4033, 4060, 4077, 4125, 4116, 4080, 4084] avg=4096.0000 max=4193 max_rank=0 min=4033 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[498, 496, 518, 524, 527, 502, 553, 506, 0, 0, 0, 0, 0, 0, 0, 0, 532, 487, 519, 462, 517, 521, 523, 558, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 516, 509, 518, 543, 525, 516, 0, 0, 0, 0, 0, 0, 0, 0, 490, 502, 495, 531, 522, 492, 527, 523, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 516, 486, 445, 493, 496, 471, 32, 5, 0, 0, 0, 0, 0, 0, 535, 495, 474, 565, 503, 528, 505, 500, 12, 0, 0, 0, 0, 0, 0, 0, 507, 492, 503, 515, 535, 535, 540, 520, 17, 9, 0, 0, 0, 0, 0, 0, 464, 522, 497, 495, 494, 510, 507, 518, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4119, 4170, 4082, 3956, 4117, 4173, 4027] avg=4096.0000 max=4173 max_rank=6 min=3956 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[501, 535, 482, 496, 480, 497, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 538, 524, 504, 527, 527, 533, 528, 552, 0, 0, 0, 0, 0, 0, 0, 0, 467, 523, 497, 516, 529, 494, 522, 468, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 483, 524, 532, 508, 485, 526, 0, 0, 0, 0, 0, 0, 0, 0, 561, 483, 524, 506, 484, 491, 510, 487, 31, 5, 0, 0, 0, 0, 0, 0, 541, 526, 515, 501, 474, 490, 505, 491, 11, 0, 0, 0, 0, 0, 0, 0, 559, 510, 548, 522, 496, 497, 506, 514, 17, 8, 0, 0, 0, 0, 0, 0, 503, 535, 530, 505, 527, 518, 500, 503, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4233, 4016, 4072, 4082, 4054, 4177, 4140] avg=4096.0000 max=4233 max_rank=1 min=3994 min_rank=0 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[529, 522, 519, 516, 528, 471, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 509, 487, 536, 507, 471, 506, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 525, 505, 493, 533, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 497, 513, 527, 560, 498, 505, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 484, 470, 503, 494, 515, 499, 36, 9, 0, 0, 0, 0, 0, 0, 505, 518, 488, 557, 491, 518, 527, 522, 16, 0, 0, 0, 0, 0, 0, 0, 511, 519, 465, 491, 480, 505, 516, 515, 20, 13, 0, 0, 0, 0, 0, 0, 513, 510, 500, 530, 528, 526, 464, 531, 15, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4020, 4124, 4154, 4055, 4142, 4035, 4129] avg=4096.0000 max=4154 max_rank=3 min=4020 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 475, 495, 498, 517, 444, 533, 543, 0, 0, 0, 0, 0, 0, 0, 0, 533, 495, 526, 517, 476, 510, 488, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 490, 523, 499, 512, 487, 521, 570, 0, 0, 0, 0, 0, 0, 0, 0, 495, 513, 477, 512, 543, 517, 525, 532, 0, 0, 0, 0, 0, 0, 0, 0, 547, 499, 548, 538, 528, 514, 519, 529, 30, 5, 0, 0, 0, 0, 0, 0, 548, 504, 501, 487, 513, 492, 493, 551, 11, 0, 0, 0, 0, 0, 0, 0, 502, 495, 503, 499, 536, 524, 483, 510, 16, 8, 0, 0, 0, 0, 0, 0, 537, 475, 534, 537, 519, 473, 470, 469, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4054, 4099, 4114, 4257, 4100, 4076, 4033] avg=4096.0000 max=4257 max_rank=4 min=4033 min_rank=7 imbalance_ratio=1.039307 max_load_reduction=28511 max_load_reduction_pct=0.870087 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[530, 489, 509, 558, 530, 492, 526, 535, 514, 522, 464, 488, 521, 497, 597, 528, 498, 522, 530, 523, 540, 531, 509, 488, 503, 517, 479, 466, 518, 479, 497, 502, 521, 499, 489, 487, 506, 506, 518, 542, 511, 528, 520, 527, 548, 531, 487, 510, 523, 516, 491, 537, 522, 510, 501, 503, 493, 476, 482, 502, 574, 526, 484, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[536, 511, 541, 499, 519, 480, 533, 507, 518, 465, 506, 517, 510, 498, 538, 491, 537, 528, 511, 500, 479, 483, 508, 541, 513, 501, 498, 480, 522, 545, 519, 531, 489, 509, 538, 525, 511, 509, 521, 511, 520, 511, 524, 536, 503, 505, 541, 497, 475, 514, 496, 518, 492, 534, 543, 492, 507, 537, 531, 534, 551, 476, 497, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[538, 541, 512, 494, 532, 518, 536, 534, 550, 506, 506, 529, 511, 477, 491, 534, 468, 530, 544, 517, 512, 478, 518, 529, 514, 523, 512, 502, 526, 513, 531, 519, 536, 451, 544, 492, 480, 508, 508, 476, 506, 500, 487, 529, 525, 501, 499, 522, 505, 507, 475, 530, 518, 530, 537, 491, 509, 535, 505, 482, 526, 512, 499, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[508, 493, 545, 532, 528, 528, 523, 519, 550, 540, 500, 522, 501, 500, 466, 521, 536, 492, 478, 523, 524, 523, 497, 513, 492, 500, 533, 535, 549, 509, 526, 547, 501, 502, 486, 534, 491, 485, 483, 503, 517, 538, 551, 500, 510, 475, 547, 502, 509, 506, 479, 531, 505, 506, 545, 522, 508, 532, 489, 500, 498, 486, 469, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[547, 523, 515, 508, 506, 484, 500, 527, 515, 509, 536, 483, 531, 514, 501, 516, 501, 512, 538, 496, 514, 537, 526, 537, 533, 507, 556, 536, 499, 518, 512, 504, 543, 484, 510, 486, 507, 514, 513, 500, 506, 513, 464, 491, 518, 524, 499, 498, 527, 517, 506, 545, 487, 503, 501, 477, 548, 519, 479, 498, 508, 534, 498, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[542, 518, 548, 541, 534, 491, 490, 563, 498, 499, 479, 493, 511, 500, 497, 482, 476, 553, 531, 518, 522, 504, 534, 491, 474, 502, 507, 521, 504, 512, 497, 528, 480, 562, 478, 530, 552, 522, 468, 513, 494, 545, 518, 497, 517, 526, 545, 517, 471, 506, 501, 531, 499, 490, 498, 528, 507, 530, 496, 527, 499, 520, 529, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[536, 510, 507, 510, 521, 497, 508, 505, 558, 506, 522, 522, 494, 531, 519, 513, 488, 509, 498, 484, 530, 514, 501, 492, 501, 482, 532, 481, 505, 526, 512, 513, 514, 511, 514, 492, 532, 501, 508, 535, 535, 521, 489, 507, 527, 513, 513, 507, 514, 508, 478, 532, 502, 501, 546, 513, 510, 486, 551, 499, 514, 495, 534, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[519, 530, 512, 508, 518, 513, 544, 494, 556, 538, 510, 490, 507, 477, 499, 528, 519, 514, 497, 519, 498, 504, 529, 496, 497, 535, 530, 498, 488, 494, 517, 514, 499, 508, 530, 510, 531, 512, 509, 498, 511, 499, 521, 500, 511, 544, 487, 525, 515, 501, 525, 478, 505, 538, 498, 530, 525, 506, 551, 487, 498, 513, 499, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[472, 489, 509, 558, 530, 492, 526, 535, 0, 0, 0, 0, 0, 0, 0, 0, 514, 522, 464, 488, 521, 497, 597, 528, 7, 0, 0, 0, 0, 0, 0, 0, 498, 519, 530, 523, 540, 531, 509, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 479, 466, 518, 479, 497, 502, 3, 1, 0, 0, 0, 0, 0, 0, 521, 499, 489, 487, 506, 506, 518, 542, 30, 0, 0, 0, 0, 0, 0, 0, 511, 528, 520, 527, 533, 531, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 523, 516, 491, 537, 522, 510, 501, 503, 9, 7, 0, 0, 0, 0, 0, 0, 493, 476, 482, 502, 574, 526, 484, 496, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4138, 4138, 3965, 4098, 4147, 4119, 4052] avg=4096.0000 max=4147 max_rank=5 min=3965 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[452, 493, 545, 532, 528, 528, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 550, 540, 500, 522, 501, 500, 466, 521, 6, 0, 0, 0, 0, 0, 0, 0, 536, 490, 478, 523, 524, 523, 497, 513, 0, 0, 0, 0, 0, 0, 0, 0, 492, 500, 533, 535, 549, 509, 526, 547, 2, 0, 0, 0, 0, 0, 0, 0, 501, 502, 486, 534, 491, 485, 483, 503, 28, 0, 0, 0, 0, 0, 0, 0, 517, 538, 551, 500, 497, 475, 547, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 479, 531, 505, 506, 545, 522, 9, 7, 0, 0, 0, 0, 0, 0, 508, 532, 489, 500, 498, 486, 469, 505, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4106, 4084, 4193, 4013, 4127, 4119, 4006] avg=4096.0000 max=4193 max_rank=3 min=4006 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 510, 507, 510, 521, 497, 508, 505, 0, 0, 0, 0, 0, 0, 0, 0, 558, 506, 522, 522, 494, 531, 519, 513, 7, 0, 0, 0, 0, 0, 0, 0, 488, 507, 498, 484, 530, 514, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 501, 482, 532, 481, 505, 526, 512, 513, 2, 1, 0, 0, 0, 0, 0, 0, 514, 511, 514, 492, 532, 501, 508, 535, 30, 0, 0, 0, 0, 0, 0, 0, 535, 521, 489, 507, 512, 513, 513, 507, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 478, 532, 502, 501, 546, 513, 9, 7, 0, 0, 0, 0, 0, 0, 510, 486, 551, 499, 514, 495, 534, 539, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4172, 4014, 4055, 4137, 4097, 4110, 4148] avg=4096.0000 max=4172 max_rank=1 min=4014 min_rank=2 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[479, 541, 512, 494, 532, 518, 536, 534, 0, 0, 0, 0, 0, 0, 0, 0, 550, 506, 506, 529, 511, 477, 491, 534, 7, 0, 0, 0, 0, 0, 0, 0, 468, 527, 544, 517, 512, 478, 518, 529, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 512, 502, 526, 513, 531, 519, 3, 1, 0, 0, 0, 0, 0, 0, 536, 451, 544, 492, 480, 508, 508, 476, 30, 0, 0, 0, 0, 0, 0, 0, 506, 500, 487, 529, 510, 501, 499, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 507, 475, 530, 518, 530, 537, 491, 9, 7, 0, 0, 0, 0, 0, 0, 509, 535, 505, 482, 526, 512, 499, 498, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4111, 4093, 4144, 4025, 4054, 4109, 4086] avg=4096.0000 max=4146 max_rank=0 min=4025 min_rank=4 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 511, 541, 499, 519, 480, 533, 507, 0, 0, 0, 0, 0, 0, 0, 0, 518, 465, 506, 517, 510, 498, 538, 491, 6, 0, 0, 0, 0, 0, 0, 0, 537, 525, 511, 500, 479, 483, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 498, 480, 522, 545, 519, 531, 3, 0, 0, 0, 0, 0, 0, 0, 489, 509, 538, 525, 511, 509, 521, 511, 30, 0, 0, 0, 0, 0, 0, 0, 520, 511, 524, 536, 490, 505, 541, 497, 0, 0, 0, 0, 0, 0, 0, 0, 475, 514, 496, 518, 492, 534, 543, 492, 9, 7, 0, 0, 0, 0, 0, 0, 507, 537, 531, 534, 551, 476, 497, 456, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4049, 4084, 4112, 4143, 4124, 4080, 4109] avg=4096.0000 max=4143 max_rank=4 min=4049 min_rank=1 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 530, 512, 508, 518, 513, 544, 494, 0, 0, 0, 0, 0, 0, 0, 0, 556, 538, 510, 490, 507, 477, 499, 528, 6, 0, 0, 0, 0, 0, 0, 0, 519, 512, 497, 519, 498, 504, 529, 496, 0, 0, 0, 0, 0, 0, 0, 0, 497, 535, 530, 498, 488, 494, 517, 514, 2, 0, 0, 0, 0, 0, 0, 0, 499, 508, 530, 510, 531, 512, 509, 498, 29, 0, 0, 0, 0, 0, 0, 0, 511, 499, 521, 500, 498, 544, 487, 525, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 525, 478, 505, 538, 498, 530, 9, 7, 0, 0, 0, 0, 0, 0, 525, 506, 551, 487, 498, 513, 499, 512, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4111, 4074, 4075, 4126, 4085, 4106, 4110] avg=4096.0000 max=4126 max_rank=4 min=4074 min_rank=2 imbalance_ratio=1.007324 max_load_reduction=28642 max_load_reduction_pct=0.874084 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[483, 518, 548, 541, 534, 491, 490, 563, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 479, 493, 511, 500, 497, 482, 6, 0, 0, 0, 0, 0, 0, 0, 476, 550, 531, 518, 522, 504, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 474, 502, 507, 521, 504, 512, 497, 528, 3, 0, 0, 0, 0, 0, 0, 0, 480, 562, 478, 530, 552, 522, 468, 513, 30, 0, 0, 0, 0, 0, 0, 0, 494, 545, 518, 497, 504, 526, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 471, 506, 501, 531, 499, 490, 498, 528, 9, 7, 0, 0, 0, 0, 0, 0, 507, 530, 496, 527, 499, 520, 529, 512, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 3965, 4126, 4048, 4135, 4146, 4040, 4140] avg=4096.0000 max=4168 max_rank=0 min=3965 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 523, 515, 508, 506, 484, 500, 527, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 536, 483, 531, 514, 501, 516, 11, 0, 0, 0, 0, 0, 0, 0, 501, 506, 538, 496, 514, 537, 526, 537, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 556, 536, 499, 518, 512, 504, 6, 5, 0, 0, 0, 0, 0, 0, 543, 484, 510, 486, 507, 514, 513, 500, 34, 0, 0, 0, 0, 0, 0, 0, 506, 513, 464, 491, 493, 524, 499, 498, 0, 0, 0, 0, 0, 0, 0, 0, 527, 517, 506, 545, 487, 503, 501, 477, 13, 9, 0, 0, 0, 0, 0, 0, 548, 519, 479, 498, 508, 534, 498, 510, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4116, 4155, 4176, 4091, 3988, 4085, 4117] avg=4096.0000 max=4176 max_rank=3 min=3988 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[527, 528, 533, 469, 500, 526, 526, 534, 506, 520, 524, 478, 504, 537, 486, 521, 502, 507, 531, 495, 535, 490, 498, 504, 522, 556, 510, 520, 520, 519, 569, 516, 464, 497, 537, 525, 523, 506, 511, 495, 451, 529, 522, 470, 524, 492, 531, 507, 509, 525, 528, 475, 557, 513, 498, 499, 519, 541, 513, 533, 482, 522, 488, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[515, 477, 495, 488, 519, 536, 532, 507, 524, 506, 515, 520, 522, 497, 471, 513, 514, 530, 516, 500, 522, 556, 526, 487, 503, 533, 501, 543, 477, 528, 501, 544, 502, 500, 517, 483, 495, 493, 526, 506, 512, 515, 542, 491, 495, 491, 539, 492, 537, 517, 540, 469, 491, 553, 492, 507, 546, 522, 509, 504, 539, 507, 507, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[508, 525, 509, 570, 506, 492, 526, 476, 505, 492, 484, 551, 505, 519, 501, 493, 527, 524, 495, 523, 505, 489, 507, 514, 506, 520, 513, 506, 518, 520, 518, 542, 501, 538, 502, 517, 501, 546, 502, 520, 494, 482, 516, 521, 519, 497, 527, 511, 518, 495, 525, 467, 499, 487, 536, 540, 518, 531, 511, 499, 508, 544, 515, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[531, 578, 497, 512, 532, 528, 523, 509, 556, 527, 510, 505, 529, 509, 507, 539, 518, 520, 517, 522, 491, 492, 506, 507, 523, 522, 502, 505, 536, 498, 484, 500, 483, 509, 486, 502, 511, 542, 528, 519, 545, 490, 519, 500, 487, 506, 521, 512, 501, 528, 523, 517, 485, 500, 478, 510, 532, 492, 512, 506, 518, 517, 444, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[530, 509, 496, 527, 497, 526, 519, 493, 519, 535, 504, 512, 533, 504, 502, 530, 514, 508, 487, 554, 538, 521, 503, 498, 499, 535, 520, 512, 510, 530, 518, 598, 555, 509, 520, 492, 518, 478, 529, 501, 512, 511, 532, 525, 501, 519, 501, 484, 483, 499, 515, 517, 507, 498, 536, 496, 515, 492, 455, 505, 490, 480, 495, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[478, 552, 497, 499, 544, 515, 530, 500, 500, 547, 521, 543, 494, 542, 524, 503, 521, 534, 543, 494, 564, 503, 543, 467, 522, 511, 514, 533, 515, 508, 481, 526, 501, 541, 522, 503, 465, 489, 507, 561, 550, 460, 470, 509, 480, 515, 549, 497, 473, 494, 468, 540, 513, 571, 523, 464, 490, 525, 512, 493, 498, 514, 494, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[550, 520, 554, 505, 478, 525, 559, 476, 507, 490, 518, 531, 520, 526, 503, 503, 495, 513, 511, 495, 486, 523, 502, 496, 516, 531, 535, 485, 501, 499, 528, 516, 503, 489, 490, 518, 507, 528, 516, 515, 484, 537, 526, 546, 512, 484, 508, 512, 508, 553, 508, 489, 508, 535, 494, 496, 519, 538, 561, 480, 503, 506, 505, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[518, 504, 484, 491, 493, 499, 484, 538, 539, 505, 487, 538, 539, 544, 500, 501, 502, 535, 546, 517, 481, 524, 522, 500, 518, 529, 495, 544, 509, 523, 508, 567, 499, 511, 468, 554, 484, 498, 531, 480, 522, 538, 519, 507, 560, 521, 498, 527, 512, 479, 531, 532, 520, 475, 525, 495, 478, 479, 540, 459, 512, 491, 535, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 528, 533, 469, 500, 526, 507, 534, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 524, 478, 504, 516, 486, 521, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 531, 495, 535, 490, 498, 504, 0, 0, 0, 0, 0, 0, 0, 0, 522, 556, 510, 520, 520, 519, 569, 464, 0, 0, 0, 0, 0, 0, 0, 0, 464, 497, 537, 525, 523, 506, 511, 495, 14, 9, 0, 0, 0, 0, 0, 0, 451, 529, 522, 470, 524, 492, 531, 507, 7, 5, 0, 0, 0, 0, 0, 0, 509, 525, 528, 475, 557, 513, 498, 499, 12, 12, 0, 0, 0, 0, 0, 0, 519, 541, 513, 533, 482, 522, 488, 469, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4055, 4057, 4180, 4081, 4038, 4128, 4105] avg=4096.0000 max=4180 max_rank=3 min=4038 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[515, 477, 495, 488, 519, 536, 513, 507, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 515, 520, 522, 477, 471, 513, 0, 0, 0, 0, 0, 0, 0, 0, 514, 525, 516, 500, 522, 556, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 501, 543, 477, 528, 501, 489, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 517, 483, 495, 493, 526, 506, 15, 9, 0, 0, 0, 0, 0, 0, 512, 515, 542, 491, 495, 491, 539, 492, 7, 5, 0, 0, 0, 0, 0, 0, 537, 517, 540, 469, 491, 553, 492, 507, 12, 11, 0, 0, 0, 0, 0, 0, 546, 522, 509, 504, 539, 507, 507, 511, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4048, 4146, 4075, 4046, 4089, 4129, 4185] avg=4096.0000 max=4185 max_rank=7 min=4046 min_rank=4 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[531, 578, 497, 512, 532, 528, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 556, 527, 510, 505, 529, 489, 507, 539, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 517, 522, 491, 492, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 502, 505, 536, 498, 484, 449, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 486, 502, 511, 542, 528, 519, 14, 9, 0, 0, 0, 0, 0, 0, 545, 490, 519, 500, 487, 506, 521, 512, 7, 5, 0, 0, 0, 0, 0, 0, 501, 528, 523, 517, 485, 500, 478, 510, 11, 11, 0, 0, 0, 0, 0, 0, 532, 492, 512, 506, 518, 517, 444, 510, 37, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4162, 4068, 4019, 4103, 4092, 4064, 4068] avg=4096.0000 max=4192 max_rank=0 min=4019 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[508, 525, 509, 570, 506, 492, 507, 476, 0, 0, 0, 0, 0, 0, 0, 0, 505, 492, 484, 551, 505, 499, 501, 493, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 495, 523, 505, 489, 507, 514, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 513, 506, 518, 520, 518, 487, 0, 0, 0, 0, 0, 0, 0, 0, 501, 538, 502, 517, 501, 546, 502, 520, 15, 9, 0, 0, 0, 0, 0, 0, 494, 482, 516, 521, 519, 497, 527, 511, 7, 5, 0, 0, 0, 0, 0, 0, 518, 495, 525, 467, 499, 487, 536, 540, 12, 11, 0, 0, 0, 0, 0, 0, 518, 531, 511, 499, 508, 544, 515, 492, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4030, 4079, 4088, 4151, 4079, 4090, 4158] avg=4096.0000 max=4158 max_rank=7 min=4030 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[478, 552, 497, 499, 544, 515, 511, 500, 0, 0, 0, 0, 0, 0, 0, 0, 500, 547, 521, 543, 494, 521, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 543, 494, 564, 503, 543, 467, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 514, 533, 515, 508, 481, 472, 0, 0, 0, 0, 0, 0, 0, 0, 501, 541, 522, 503, 465, 489, 507, 561, 15, 9, 0, 0, 0, 0, 0, 0, 550, 460, 470, 509, 480, 515, 549, 497, 7, 5, 0, 0, 0, 0, 0, 0, 473, 494, 468, 540, 513, 571, 523, 464, 12, 12, 0, 0, 0, 0, 0, 0, 490, 525, 512, 493, 498, 514, 494, 509, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4153, 4164, 4056, 4113, 4042, 4070, 4074] avg=4096.0000 max=4164 max_rank=2 min=4042 min_rank=5 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 520, 554, 505, 478, 525, 540, 476, 0, 0, 0, 0, 0, 0, 0, 0, 507, 490, 518, 531, 520, 505, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 511, 495, 486, 523, 502, 496, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 535, 485, 501, 499, 528, 464, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 490, 518, 507, 528, 516, 515, 14, 9, 0, 0, 0, 0, 0, 0, 484, 537, 526, 546, 512, 484, 508, 512, 7, 5, 0, 0, 0, 0, 0, 0, 508, 553, 508, 489, 508, 535, 494, 496, 12, 12, 0, 0, 0, 0, 0, 0, 519, 538, 561, 480, 503, 506, 505, 493, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4077, 4016, 4059, 4089, 4121, 4115, 4143] avg=4096.0000 max=4148 max_rank=0 min=4016 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 496, 527, 497, 526, 493, 493, 0, 0, 0, 0, 0, 0, 0, 0, 519, 535, 504, 512, 533, 476, 502, 530, 0, 0, 0, 0, 0, 0, 0, 0, 514, 501, 487, 554, 538, 521, 503, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 535, 520, 512, 510, 530, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0, 555, 509, 520, 492, 518, 478, 529, 501, 20, 13, 0, 0, 0, 0, 0, 0, 512, 511, 532, 525, 501, 519, 501, 484, 12, 7, 0, 0, 0, 0, 0, 0, 483, 499, 515, 517, 507, 498, 536, 496, 14, 15, 0, 0, 0, 0, 0, 0, 515, 492, 455, 505, 490, 480, 495, 517, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4111, 4116, 4156, 4135, 4104, 4080, 3995] avg=4096.0000 max=4156 max_rank=3 min=3995 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 504, 484, 491, 493, 499, 467, 538, 0, 0, 0, 0, 0, 0, 0, 0, 539, 505, 487, 538, 539, 523, 500, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 530, 546, 517, 481, 524, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 495, 544, 509, 523, 508, 509, 0, 0, 0, 0, 0, 0, 0, 0, 499, 511, 468, 554, 484, 498, 531, 480, 16, 9, 0, 0, 0, 0, 0, 0, 522, 538, 519, 507, 560, 521, 498, 527, 6, 5, 0, 0, 0, 0, 0, 0, 512, 479, 531, 532, 520, 475, 525, 495, 11, 12, 0, 0, 0, 0, 0, 0, 478, 479, 540, 459, 512, 491, 535, 504, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3994, 4132, 4122, 4135, 4050, 4203, 4092, 4040] avg=4096.0000 max=4203 max_rank=5 min=3994 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4352, 4096, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[505, 485, 509, 507, 498, 493, 537, 508, 515, 559, 529, 528, 521, 515, 527, 513, 535, 519, 508, 527, 538, 525, 548, 497, 532, 544, 471, 501, 539, 495, 504, 548, 500, 501, 544, 494, 524, 498, 455, 531, 491, 520, 513, 523, 511, 492, 488, 491, 517, 502, 479, 530, 518, 538, 541, 494, 491, 479, 488, 499, 514, 528, 491, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[523, 529, 553, 539, 530, 492, 512, 499, 526, 497, 494, 496, 539, 504, 515, 567, 498, 529, 487, 518, 522, 525, 499, 523, 519, 518, 526, 495, 487, 503, 526, 520, 493, 518, 520, 516, 479, 488, 516, 521, 519, 509, 475, 535, 493, 540, 530, 522, 502, 498, 502, 539, 485, 492, 541, 545, 506, 510, 477, 489, 482, 534, 498, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[502, 550, 471, 523, 487, 502, 491, 514, 539, 499, 501, 510, 545, 506, 507, 489, 516, 556, 479, 509, 530, 508, 518, 500, 522, 533, 498, 519, 498, 513, 470, 501, 521, 551, 463, 500, 524, 519, 519, 511, 507, 511, 504, 501, 530, 486, 514, 526, 484, 506, 512, 519, 530, 518, 537, 506, 508, 543, 536, 524, 496, 512, 540, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[474, 561, 542, 539, 517, 510, 543, 512, 515, 544, 496, 491, 493, 549, 545, 478, 491, 497, 487, 503, 507, 501, 516, 511, 533, 524, 489, 486, 497, 555, 498, 524, 498, 513, 521, 481, 503, 498, 529, 497, 530, 508, 508, 525, 532, 491, 530, 527, 525, 542, 501, 467, 527, 487, 513, 486, 526, 527, 508, 507, 567, 495, 486, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[562, 513, 519, 487, 527, 495, 504, 500, 511, 541, 487, 508, 505, 505, 500, 517, 513, 494, 518, 529, 538, 499, 519, 521, 498, 508, 532, 513, 550, 551, 516, 533, 531, 463, 499, 539, 544, 476, 475, 521, 507, 468, 535, 511, 516, 513, 521, 462, 517, 508, 500, 518, 528, 504, 523, 505, 501, 495, 493, 526, 537, 520, 518, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[495, 542, 512, 514, 519, 505, 514, 525, 502, 539, 537, 535, 514, 552, 548, 469, 517, 520, 528, 505, 511, 508, 497, 524, 515, 524, 510, 543, 474, 468, 503, 544, 512, 506, 521, 520, 523, 503, 488, 483, 472, 516, 531, 489, 539, 530, 505, 506, 506, 519, 527, 494, 526, 502, 503, 497, 558, 518, 505, 499, 465, 517, 484, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[470, 501, 485, 476, 495, 543, 515, 520, 515, 525, 541, 510, 496, 517, 497, 493, 543, 503, 500, 539, 520, 547, 506, 538, 493, 529, 549, 542, 535, 499, 512, 491, 524, 529, 507, 552, 504, 490, 497, 519, 504, 484, 500, 511, 583, 516, 524, 517, 473, 517, 474, 546, 518, 498, 515, 485, 507, 509, 498, 523, 490, 488, 488, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[517, 515, 493, 526, 525, 528, 503, 519, 525, 494, 511, 511, 506, 545, 516, 486, 520, 542, 496, 485, 548, 522, 513, 470, 532, 530, 483, 511, 512, 536, 538, 505, 495, 542, 561, 535, 499, 506, 507, 519, 523, 489, 505, 494, 525, 500, 492, 500, 493, 508, 502, 489, 510, 510, 540, 509, 494, 548, 477, 503, 524, 480, 507, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 509, 493, 526, 525, 528, 503, 519, 0, 0, 0, 0, 0, 0, 0, 0, 525, 466, 511, 511, 506, 545, 516, 486, 0, 0, 0, 0, 0, 0, 0, 0, 520, 542, 496, 485, 525, 522, 513, 470, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 483, 511, 512, 536, 538, 505, 0, 0, 0, 0, 0, 0, 0, 0, 495, 542, 561, 535, 499, 506, 507, 519, 13, 3, 0, 0, 0, 0, 0, 0, 523, 489, 505, 494, 525, 500, 492, 500, 6, 5, 0, 0, 0, 0, 0, 0, 493, 508, 502, 489, 510, 510, 540, 509, 15, 0, 0, 0, 0, 0, 0, 0, 494, 548, 477, 503, 524, 480, 507, 519, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4066, 4073, 4123, 4180, 4039, 4076, 4091] avg=4096.0000 max=4180 max_rank=4 min=4039 min_rank=5 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 536, 512, 514, 519, 505, 514, 525, 0, 0, 0, 0, 0, 0, 0, 0, 502, 508, 537, 535, 514, 552, 548, 469, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 528, 505, 489, 508, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 510, 543, 474, 468, 503, 544, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 521, 520, 523, 503, 488, 483, 13, 3, 0, 0, 0, 0, 0, 0, 472, 516, 531, 489, 539, 530, 505, 506, 6, 5, 0, 0, 0, 0, 0, 0, 506, 519, 527, 494, 526, 502, 503, 497, 14, 0, 0, 0, 0, 0, 0, 0, 558, 518, 505, 499, 465, 517, 484, 491, 31, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4165, 4088, 4057, 4072, 4099, 4088, 4079] avg=4096.0000 max=4165 max_rank=1 min=4057 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 523, 553, 539, 530, 492, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 526, 469, 494, 496, 539, 504, 515, 567, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 487, 518, 500, 525, 499, 523, 0, 0, 0, 0, 0, 0, 0, 0, 519, 494, 526, 495, 487, 503, 526, 520, 0, 0, 0, 0, 0, 0, 0, 0, 493, 518, 520, 516, 479, 488, 516, 521, 13, 3, 0, 0, 0, 0, 0, 0, 519, 509, 475, 535, 493, 540, 530, 522, 6, 5, 0, 0, 0, 0, 0, 0, 502, 498, 502, 539, 485, 492, 541, 545, 14, 0, 0, 0, 0, 0, 0, 0, 506, 510, 477, 489, 482, 534, 498, 484, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4110, 4079, 4070, 4067, 4134, 4118, 4019] avg=4096.0000 max=4171 max_rank=0 min=4019 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[502, 544, 471, 523, 487, 502, 491, 514, 0, 0, 0, 0, 0, 0, 0, 0, 539, 471, 501, 510, 545, 506, 507, 489, 0, 0, 0, 0, 0, 0, 0, 0, 516, 556, 479, 509, 507, 508, 518, 500, 0, 0, 0, 0, 0, 0, 0, 0, 522, 509, 498, 519, 498, 513, 470, 501, 0, 0, 0, 0, 0, 0, 0, 0, 521, 551, 463, 500, 524, 519, 519, 511, 13, 3, 0, 0, 0, 0, 0, 0, 507, 511, 504, 501, 530, 486, 514, 526, 6, 5, 0, 0, 0, 0, 0, 0, 484, 506, 512, 519, 530, 518, 537, 506, 15, 0, 0, 0, 0, 0, 0, 0, 508, 543, 536, 524, 496, 512, 540, 504, 28, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4068, 4093, 4030, 4124, 4090, 4127, 4202] avg=4096.0000 max=4202 max_rank=7 min=4030 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16543 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 479, 509, 507, 498, 493, 537, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 529, 528, 521, 515, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 519, 508, 527, 515, 525, 548, 497, 0, 0, 0, 0, 0, 0, 0, 0, 532, 519, 471, 501, 539, 495, 504, 548, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 544, 494, 524, 498, 455, 531, 13, 3, 0, 0, 0, 0, 0, 0, 491, 520, 513, 523, 511, 492, 488, 491, 6, 5, 0, 0, 0, 0, 0, 0, 517, 502, 479, 530, 518, 538, 541, 494, 15, 0, 0, 0, 0, 0, 0, 0, 491, 479, 488, 499, 514, 528, 491, 503, 32, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4175, 4174, 4109, 4063, 4040, 4134, 4037] avg=4096.0000 max=4175 max_rank=1 min=4036 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[474, 554, 542, 539, 517, 510, 543, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 496, 491, 493, 549, 545, 478, 0, 0, 0, 0, 0, 0, 0, 0, 491, 497, 487, 503, 485, 501, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 500, 489, 486, 497, 555, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 521, 481, 503, 498, 529, 497, 13, 3, 0, 0, 0, 0, 0, 0, 530, 508, 508, 525, 532, 491, 530, 527, 7, 5, 0, 0, 0, 0, 0, 0, 525, 542, 501, 467, 527, 487, 513, 486, 14, 0, 0, 0, 0, 0, 0, 0, 526, 527, 508, 507, 567, 495, 486, 485, 31, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4191, 4080, 3991, 4082, 4056, 4163, 4062, 4143] avg=4096.0000 max=4191 max_rank=0 min=3991 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 491, 485, 476, 495, 543, 515, 520, 0, 0, 0, 0, 0, 0, 0, 0, 515, 492, 541, 510, 496, 517, 497, 493, 0, 0, 0, 0, 0, 0, 0, 0, 543, 503, 500, 539, 486, 547, 506, 538, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 549, 542, 535, 499, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 507, 552, 504, 490, 497, 519, 16, 5, 0, 0, 0, 0, 0, 0, 504, 484, 500, 511, 583, 516, 524, 517, 10, 10, 0, 0, 0, 0, 0, 0, 473, 517, 474, 546, 518, 498, 515, 485, 19, 0, 0, 0, 0, 0, 0, 0, 507, 509, 498, 523, 490, 488, 488, 533, 33, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4061, 4162, 4119, 4143, 4159, 4045, 4084] avg=4096.0000 max=4162 max_rank=2 min=3995 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[562, 507, 519, 487, 527, 495, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 487, 508, 505, 505, 500, 517, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 518, 529, 515, 499, 519, 521, 0, 0, 0, 0, 0, 0, 0, 0, 498, 485, 532, 513, 550, 551, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 531, 463, 499, 539, 544, 476, 475, 521, 12, 3, 0, 0, 0, 0, 0, 0, 507, 468, 535, 511, 516, 513, 521, 462, 6, 5, 0, 0, 0, 0, 0, 0, 517, 508, 500, 518, 528, 504, 523, 505, 15, 0, 0, 0, 0, 0, 0, 0, 501, 495, 493, 526, 537, 520, 518, 481, 31, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4043, 4108, 4178, 4063, 4044, 4118, 4113] avg=4096.0000 max=4178 max_rank=3 min=4043 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4352, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[549, 499, 515, 531, 489, 503, 493, 537, 533, 510, 489, 471, 518, 552, 530, 515, 512, 510, 462, 520, 535, 504, 509, 561, 484, 533, 491, 512, 512, 497, 483, 524, 520, 511, 521, 497, 545, 529, 488, 508, 509, 502, 518, 521, 562, 483, 509, 490, 482, 536, 524, 522, 519, 512, 535, 509, 474, 517, 445, 564, 513, 469, 523, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[501, 526, 485, 504, 508, 506, 553, 509, 530, 505, 505, 536, 538, 514, 527, 515, 513, 512, 521, 529, 529, 498, 532, 507, 497, 508, 509, 508, 529, 491, 523, 492, 501, 509, 472, 539, 505, 497, 546, 504, 541, 535, 514, 553, 489, 494, 504, 488, 508, 532, 506, 468, 508, 549, 495, 495, 491, 495, 477, 510, 524, 543, 518, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[519, 529, 501, 513, 499, 504, 529, 537, 518, 520, 511, 532, 513, 515, 518, 490, 511, 531, 523, 551, 506, 516, 523, 534, 549, 501, 495, 511, 558, 512, 519, 516, 500, 502, 500, 481, 490, 523, 516, 520, 524, 518, 488, 503, 499, 516, 522, 516, 492, 533, 515, 523, 530, 500, 491, 480, 505, 464, 508, 481, 505, 513, 501, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[515, 508, 547, 532, 517, 511, 518, 510, 502, 506, 468, 512, 497, 501, 514, 506, 524, 513, 482, 524, 537, 524, 524, 514, 525, 519, 533, 510, 543, 507, 524, 511, 472, 516, 520, 462, 528, 493, 501, 524, 510, 562, 507, 500, 521, 521, 522, 508, 528, 511, 516, 494, 490, 516, 479, 530, 512, 513, 485, 493, 513, 517, 496, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[515, 531, 547, 524, 526, 532, 524, 534, 510, 508, 505, 488, 515, 510, 474, 469, 489, 524, 529, 539, 536, 513, 486, 528, 524, 534, 533, 512, 511, 518, 542, 531, 559, 480, 465, 551, 499, 497, 526, 490, 473, 471, 498, 503, 486, 525, 512, 520, 491, 496, 497, 543, 506, 471, 487, 514, 529, 507, 482, 531, 509, 504, 547, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[474, 500, 528, 533, 516, 508, 568, 508, 503, 526, 510, 517, 527, 484, 514, 532, 532, 557, 493, 516, 456, 538, 476, 514, 547, 506, 484, 505, 529, 501, 505, 536, 574, 539, 534, 506, 506, 505, 560, 510, 521, 477, 507, 466, 511, 529, 525, 475, 514, 510, 498, 513, 496, 500, 539, 536, 500, 517, 499, 513, 481, 496, 476, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[504, 485, 537, 521, 520, 537, 511, 530, 484, 562, 553, 510, 541, 522, 499, 513, 517, 525, 545, 516, 509, 503, 489, 540, 572, 523, 531, 512, 511, 501, 518, 526, 508, 513, 504, 518, 456, 537, 500, 508, 493, 495, 517, 494, 515, 533, 508, 505, 499, 489, 478, 517, 508, 513, 530, 481, 489, 495, 494, 528, 512, 507, 501, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[498, 512, 483, 505, 495, 523, 546, 551, 527, 529, 527, 541, 518, 528, 489, 522, 456, 476, 499, 527, 529, 489, 493, 466, 505, 540, 500, 540, 504, 463, 510, 510, 546, 508, 505, 512, 484, 499, 500, 533, 554, 529, 514, 516, 531, 528, 547, 529, 529, 501, 504, 499, 514, 527, 534, 498, 532, 511, 497, 499, 453, 537, 487, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[549, 499, 515, 531, 489, 503, 452, 537, 0, 0, 0, 0, 0, 0, 0, 0, 533, 510, 489, 471, 507, 552, 530, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 462, 506, 535, 504, 509, 561, 0, 0, 0, 0, 0, 0, 0, 0, 457, 533, 491, 512, 512, 497, 483, 524, 0, 0, 0, 0, 0, 0, 0, 0, 520, 511, 521, 497, 545, 529, 488, 508, 10, 0, 0, 0, 0, 0, 0, 0, 509, 502, 518, 521, 562, 483, 509, 490, 1, 0, 0, 0, 0, 0, 0, 0, 482, 536, 524, 522, 519, 512, 535, 509, 14, 10, 0, 0, 0, 0, 0, 0, 474, 517, 445, 564, 513, 469, 523, 528, 41, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4107, 4099, 4009, 4129, 4095, 4163, 4091] avg=4096.0000 max=4163 max_rank=6 min=4009 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16478 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 529, 501, 513, 499, 504, 485, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 511, 532, 502, 515, 518, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 523, 535, 506, 516, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 495, 511, 558, 512, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 500, 481, 490, 523, 516, 520, 10, 1, 0, 0, 0, 0, 0, 0, 524, 518, 488, 503, 499, 516, 522, 516, 1, 0, 0, 0, 0, 0, 0, 0, 492, 533, 515, 523, 530, 500, 491, 480, 15, 11, 0, 0, 0, 0, 0, 0, 505, 464, 508, 481, 505, 513, 501, 505, 44, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4106, 4179, 4130, 4043, 4087, 4090, 4046] avg=4096.0000 max=4179 max_rank=2 min=4043 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[474, 500, 528, 533, 516, 508, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 510, 517, 515, 484, 514, 532, 0, 0, 0, 0, 0, 0, 0, 0, 532, 557, 493, 502, 456, 538, 476, 514, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 484, 505, 529, 501, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 574, 539, 534, 506, 506, 505, 560, 510, 11, 0, 0, 0, 0, 0, 0, 0, 521, 477, 507, 466, 511, 529, 525, 475, 1, 0, 0, 0, 0, 0, 0, 0, 514, 510, 498, 513, 496, 500, 539, 536, 14, 11, 0, 0, 0, 0, 0, 0, 500, 517, 499, 513, 481, 496, 476, 492, 47, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4101, 4068, 4082, 4245, 4012, 4131, 4041] avg=4096.0000 max=4245 max_rank=4 min=4012 min_rank=5 imbalance_ratio=1.036377 max_load_reduction=28523 max_load_reduction_pct=0.870453 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 531, 547, 524, 526, 532, 481, 534, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 505, 488, 504, 510, 474, 469, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 529, 523, 536, 513, 486, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 534, 533, 512, 511, 518, 542, 531, 0, 0, 0, 0, 0, 0, 0, 0, 559, 480, 465, 551, 499, 497, 526, 490, 10, 1, 0, 0, 0, 0, 0, 0, 473, 471, 498, 503, 486, 525, 512, 520, 1, 0, 0, 0, 0, 0, 0, 0, 491, 496, 497, 543, 506, 471, 487, 514, 15, 10, 0, 0, 0, 0, 0, 0, 529, 507, 482, 531, 509, 504, 547, 538, 43, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 3968, 4128, 4176, 4078, 3989, 4030, 4209] avg=4096.0000 max=4209 max_rank=7 min=3968 min_rank=1 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 508, 547, 532, 517, 511, 475, 510, 0, 0, 0, 0, 0, 0, 0, 0, 502, 506, 468, 512, 486, 501, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 482, 510, 537, 524, 524, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 533, 510, 543, 507, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 472, 516, 520, 462, 528, 493, 501, 524, 10, 0, 0, 0, 0, 0, 0, 0, 510, 562, 507, 500, 521, 521, 522, 508, 1, 0, 0, 0, 0, 0, 0, 0, 528, 511, 516, 494, 490, 516, 479, 530, 14, 10, 0, 0, 0, 0, 0, 0, 512, 513, 485, 493, 513, 517, 496, 530, 43, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 3995, 4128, 4143, 4026, 4152, 4088, 4121] avg=4096.0000 max=4152 max_rank=5 min=3995 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[504, 485, 537, 521, 520, 537, 469, 530, 0, 0, 0, 0, 0, 0, 0, 0, 484, 562, 553, 510, 529, 522, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 545, 502, 509, 503, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0, 541, 523, 531, 512, 511, 501, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 504, 518, 456, 537, 500, 508, 11, 0, 0, 0, 0, 0, 0, 0, 493, 495, 517, 494, 515, 533, 508, 505, 1, 0, 0, 0, 0, 0, 0, 0, 499, 489, 478, 517, 508, 513, 530, 481, 14, 11, 0, 0, 0, 0, 0, 0, 489, 495, 494, 528, 512, 507, 501, 456, 42, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4172, 4130, 4163, 4055, 4061, 4040, 4044] avg=4096.0000 max=4172 max_rank=1 min=4040 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 526, 485, 504, 508, 506, 507, 509, 0, 0, 0, 0, 0, 0, 0, 0, 530, 505, 505, 536, 526, 514, 527, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 512, 521, 513, 529, 498, 532, 507, 0, 0, 0, 0, 0, 0, 0, 0, 469, 508, 509, 508, 529, 491, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 472, 539, 505, 497, 546, 504, 11, 1, 0, 0, 0, 0, 0, 0, 541, 535, 514, 553, 489, 494, 504, 488, 1, 0, 0, 0, 0, 0, 0, 0, 508, 532, 506, 468, 508, 549, 495, 495, 15, 10, 0, 0, 0, 0, 0, 0, 491, 495, 477, 510, 524, 543, 518, 498, 46, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4158, 4125, 4029, 4085, 4119, 4086, 4120] avg=4096.0000 max=4158 max_rank=1 min=4029 min_rank=3 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 512, 483, 505, 495, 523, 497, 551, 0, 0, 0, 0, 0, 0, 0, 0, 527, 529, 527, 541, 498, 528, 489, 522, 0, 0, 0, 0, 0, 0, 0, 0, 456, 476, 499, 503, 529, 489, 493, 466, 0, 0, 0, 0, 0, 0, 0, 0, 469, 540, 500, 540, 504, 463, 510, 510, 0, 0, 0, 0, 0, 0, 0, 0, 546, 508, 505, 512, 484, 499, 500, 533, 15, 5, 0, 0, 0, 0, 0, 0, 554, 529, 514, 516, 531, 528, 547, 529, 5, 0, 0, 0, 0, 0, 0, 0, 529, 501, 504, 499, 514, 527, 534, 498, 19, 15, 0, 0, 0, 0, 0, 0, 532, 511, 497, 499, 453, 537, 487, 510, 49, 21, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4161, 3911, 4036, 4107, 4253, 4140, 4096] avg=4096.0000 max=4253 max_rank=5 min=3911 min_rank=2 imbalance_ratio=1.038330 max_load_reduction=28515 max_load_reduction_pct=0.870209 moved_to_echo=16596 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 3968, 4224, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[521, 505, 542, 548, 501, 531, 530, 514, 535, 518, 536, 533, 535, 524, 524, 512, 548, 503, 526, 516, 459, 503, 484, 550, 493, 512, 507, 523, 506, 500, 530, 493, 482, 513, 523, 516, 503, 487, 542, 474, 522, 493, 548, 532, 491, 473, 510, 517, 485, 495, 501, 555, 527, 476, 513, 506, 504, 509, 494, 539, 482, 494, 513, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[543, 526, 519, 506, 502, 525, 551, 542, 494, 494, 471, 540, 521, 519, 507, 530, 526, 536, 531, 514, 505, 523, 500, 533, 496, 523, 481, 456, 496, 515, 489, 530, 516, 484, 501, 502, 483, 500, 503, 518, 513, 507, 534, 515, 524, 487, 509, 492, 506, 550, 493, 518, 532, 532, 495, 530, 524, 545, 497, 510, 458, 560, 506, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[470, 480, 498, 500, 525, 504, 511, 525, 514, 532, 531, 530, 517, 497, 521, 516, 499, 504, 546, 514, 500, 522, 505, 494, 545, 519, 511, 476, 503, 507, 548, 484, 489, 492, 522, 528, 521, 514, 511, 496, 500, 532, 519, 492, 492, 511, 550, 483, 505, 498, 508, 551, 490, 510, 526, 547, 530, 495, 480, 538, 542, 519, 511, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[524, 517, 499, 524, 531, 568, 530, 516, 534, 467, 499, 535, 488, 479, 471, 507, 526, 512, 498, 499, 527, 489, 501, 520, 479, 520, 489, 506, 502, 475, 527, 550, 484, 526, 558, 500, 554, 526, 510, 499, 486, 482, 519, 531, 530, 531, 510, 508, 492, 530, 510, 500, 533, 467, 517, 516, 497, 502, 492, 498, 559, 535, 529, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[521, 513, 509, 520, 525, 514, 517, 508, 528, 504, 515, 513, 461, 510, 515, 533, 518, 520, 481, 539, 496, 513, 549, 522, 495, 481, 495, 542, 511, 500, 529, 476, 510, 547, 528, 508, 529, 521, 490, 507, 503, 476, 510, 485, 533, 510, 557, 523, 480, 527, 493, 529, 508, 567, 521, 497, 476, 535, 478, 492, 533, 484, 513, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[525, 541, 466, 529, 550, 508, 556, 492, 538, 501, 532, 508, 483, 517, 530, 491, 549, 486, 487, 489, 512, 534, 541, 528, 485, 503, 503, 494, 502, 508, 539, 520, 525, 518, 511, 525, 514, 506, 528, 526, 494, 506, 497, 504, 513, 516, 526, 529, 481, 492, 490, 471, 500, 515, 544, 516, 534, 503, 500, 519, 498, 516, 508, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[506, 523, 543, 501, 544, 541, 520, 501, 502, 511, 508, 529, 518, 511, 530, 548, 531, 492, 518, 506, 476, 556, 562, 517, 521, 530, 541, 497, 495, 510, 490, 520, 500, 478, 531, 508, 490, 490, 502, 466, 501, 537, 515, 520, 498, 502, 492, 548, 502, 489, 481, 491, 495, 524, 489, 524, 486, 477, 559, 489, 498, 519, 517, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[502, 517, 517, 511, 506, 506, 471, 530, 524, 535, 506, 515, 486, 514, 497, 542, 506, 509, 503, 509, 474, 493, 496, 522, 505, 534, 514, 456, 546, 492, 505, 521, 501, 515, 503, 537, 509, 498, 482, 534, 542, 549, 527, 515, 524, 507, 508, 522, 494, 517, 498, 486, 557, 524, 511, 565, 482, 520, 479, 506, 505, 521, 510, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 505, 542, 548, 501, 482, 530, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 536, 519, 535, 524, 524, 512, 0, 0, 0, 0, 0, 0, 0, 0, 539, 503, 526, 516, 459, 503, 484, 550, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 507, 523, 506, 500, 530, 493, 40, 0, 0, 0, 0, 0, 0, 0, 482, 513, 523, 516, 503, 487, 542, 474, 9, 9, 0, 0, 0, 0, 0, 0, 522, 493, 540, 532, 491, 473, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 501, 555, 527, 476, 513, 506, 8, 1, 0, 0, 0, 0, 0, 0, 504, 509, 494, 539, 482, 494, 513, 487, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4203, 4080, 4104, 4058, 4078, 4067, 4035] avg=4096.0000 max=4203 max_rank=1 min=4035 min_rank=7 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16238 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[506, 523, 543, 501, 544, 492, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 508, 515, 518, 511, 530, 548, 0, 0, 0, 0, 0, 0, 0, 0, 522, 492, 518, 506, 476, 556, 562, 517, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 541, 497, 495, 510, 490, 520, 40, 0, 0, 0, 0, 0, 0, 0, 500, 478, 531, 508, 490, 490, 502, 466, 9, 9, 0, 0, 0, 0, 0, 0, 501, 537, 508, 520, 498, 502, 492, 548, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 481, 491, 495, 524, 489, 524, 7, 1, 0, 0, 0, 0, 0, 0, 486, 477, 559, 489, 498, 519, 517, 552, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4143, 4149, 4144, 3983, 4106, 4003, 4110] avg=4096.0000 max=4149 max_rank=2 min=3983 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16202 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 517, 499, 524, 531, 516, 530, 516, 0, 0, 0, 0, 0, 0, 0, 0, 534, 467, 499, 521, 488, 479, 471, 507, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 498, 499, 527, 489, 501, 520, 0, 0, 0, 0, 0, 0, 0, 0, 479, 520, 489, 506, 502, 475, 527, 550, 42, 0, 0, 0, 0, 0, 0, 0, 484, 526, 558, 500, 554, 526, 510, 499, 9, 10, 0, 0, 0, 0, 0, 0, 486, 482, 512, 531, 530, 531, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 492, 530, 510, 500, 533, 467, 517, 516, 7, 1, 0, 0, 0, 0, 0, 0, 497, 502, 492, 498, 559, 535, 529, 528, 8, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 3966, 4063, 4090, 4176, 4090, 4073, 4153] avg=4096.0000 max=4176 max_rank=4 min=3966 min_rank=1 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[470, 480, 498, 500, 525, 457, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 514, 532, 531, 516, 517, 497, 521, 516, 0, 0, 0, 0, 0, 0, 0, 0, 491, 504, 546, 514, 500, 522, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 511, 476, 503, 507, 548, 484, 38, 0, 0, 0, 0, 0, 0, 0, 489, 492, 522, 528, 521, 514, 511, 496, 9, 9, 0, 0, 0, 0, 0, 0, 500, 532, 512, 492, 492, 511, 550, 483, 0, 0, 0, 0, 0, 0, 0, 0, 505, 498, 508, 551, 490, 510, 526, 547, 7, 1, 0, 0, 0, 0, 0, 0, 530, 495, 480, 538, 542, 519, 511, 518, 7, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4144, 4076, 4131, 4091, 4072, 4143, 4145] avg=4096.0000 max=4145 max_rank=7 min=3966 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 541, 466, 529, 550, 461, 556, 492, 0, 0, 0, 0, 0, 0, 0, 0, 538, 501, 532, 495, 483, 517, 530, 491, 0, 0, 0, 0, 0, 0, 0, 0, 540, 486, 487, 489, 512, 534, 541, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 503, 503, 494, 502, 508, 539, 520, 38, 0, 0, 0, 0, 0, 0, 0, 525, 518, 511, 525, 514, 506, 528, 526, 9, 9, 0, 0, 0, 0, 0, 0, 494, 506, 490, 504, 513, 516, 526, 529, 0, 0, 0, 0, 0, 0, 0, 0, 481, 492, 490, 471, 500, 515, 544, 516, 7, 1, 0, 0, 0, 0, 0, 0, 534, 503, 500, 519, 498, 516, 508, 496, 8, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4087, 4117, 4092, 4171, 4078, 4017, 4086] avg=4096.0000 max=4171 max_rank=4 min=4017 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[502, 517, 517, 511, 506, 459, 471, 530, 0, 0, 0, 0, 0, 0, 0, 0, 524, 535, 506, 502, 486, 514, 497, 542, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 503, 509, 474, 493, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 514, 456, 546, 492, 505, 521, 38, 0, 0, 0, 0, 0, 0, 0, 501, 515, 503, 537, 509, 498, 482, 534, 9, 9, 0, 0, 0, 0, 0, 0, 542, 549, 519, 515, 524, 507, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 517, 498, 486, 557, 524, 511, 565, 8, 1, 0, 0, 0, 0, 0, 0, 482, 520, 479, 506, 505, 521, 510, 556, 8, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4106, 4003, 4111, 4097, 4186, 4161, 4091] avg=4096.0000 max=4186 max_rank=5 min=4003 min_rank=2 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16535 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 513, 509, 520, 525, 467, 517, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 515, 500, 461, 510, 515, 533, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 481, 539, 496, 513, 549, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 495, 542, 511, 500, 529, 476, 38, 0, 0, 0, 0, 0, 0, 0, 510, 547, 528, 508, 529, 521, 490, 507, 9, 9, 0, 0, 0, 0, 0, 0, 503, 476, 503, 485, 533, 510, 557, 523, 0, 0, 0, 0, 0, 0, 0, 0, 480, 527, 493, 529, 508, 567, 521, 497, 7, 1, 0, 0, 0, 0, 0, 0, 476, 535, 478, 492, 533, 484, 513, 525, 8, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4066, 4129, 4067, 4158, 4090, 4130, 4048] avg=4096.0000 max=4158 max_rank=4 min=4048 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 526, 519, 506, 502, 470, 551, 542, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 471, 517, 521, 519, 507, 530, 0, 0, 0, 0, 0, 0, 0, 0, 509, 536, 531, 514, 505, 523, 500, 533, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 481, 456, 496, 515, 489, 530, 43, 0, 0, 0, 0, 0, 0, 0, 516, 484, 501, 502, 483, 500, 503, 518, 15, 12, 0, 0, 0, 0, 0, 0, 513, 507, 521, 515, 524, 487, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 550, 493, 518, 532, 532, 495, 530, 13, 5, 0, 0, 0, 0, 0, 0, 524, 545, 497, 510, 458, 560, 506, 480, 12, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4053, 4151, 4029, 4034, 4068, 4174, 4100] avg=4096.0000 max=4174 max_rank=6 min=4029 min_rank=3 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[3968, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 3968, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[540, 548, 482, 506, 503, 480, 542, 541, 532, 531, 513, 549, 507, 471, 501, 487, 530, 483, 486, 496, 505, 533, 531, 540, 529, 480, 494, 489, 498, 544, 497, 445, 516, 537, 514, 517, 534, 495, 504, 529, 492, 536, 527, 522, 511, 506, 507, 504, 500, 525, 521, 509, 509, 534, 533, 504, 503, 522, 501, 500, 517, 479, 575, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[474, 510, 497, 519, 496, 517, 518, 506, 525, 531, 489, 493, 531, 536, 473, 512, 502, 507, 493, 493, 494, 534, 557, 500, 512, 582, 483, 502, 545, 514, 509, 511, 533, 529, 529, 494, 552, 517, 541, 487, 487, 522, 508, 503, 513, 509, 528, 518, 482, 550, 490, 522, 496, 509, 535, 513, 529, 501, 534, 488, 485, 499, 457, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[526, 534, 493, 531, 486, 502, 521, 531, 502, 513, 509, 517, 523, 533, 520, 531, 468, 533, 505, 529, 507, 541, 518, 495, 525, 491, 477, 497, 487, 524, 538, 520, 585, 527, 548, 520, 485, 513, 525, 515, 487, 536, 517, 503, 489, 473, 508, 537, 536, 555, 494, 456, 504, 509, 502, 503, 480, 530, 472, 530, 488, 519, 493, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[487, 483, 504, 543, 502, 471, 549, 538, 465, 497, 542, 517, 528, 531, 510, 541, 512, 472, 527, 482, 539, 498, 507, 482, 497, 505, 502, 492, 471, 484, 522, 518, 491, 520, 473, 524, 500, 536, 502, 529, 490, 539, 516, 529, 504, 523, 542, 515, 539, 529, 497, 551, 536, 535, 493, 502, 552, 455, 509, 500, 563, 490, 537, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[486, 488, 546, 505, 502, 505, 519, 497, 507, 522, 502, 514, 497, 539, 522, 554, 542, 536, 534, 533, 557, 525, 538, 533, 515, 512, 471, 509, 519, 499, 504, 518, 507, 505, 512, 503, 508, 487, 471, 525, 472, 484, 513, 548, 523, 502, 503, 503, 515, 536, 527, 521, 507, 536, 511, 501, 518, 499, 517, 492, 471, 508, 492, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[540, 532, 551, 496, 502, 530, 499, 523, 506, 526, 496, 520, 510, 501, 482, 511, 505, 511, 483, 514, 528, 465, 540, 508, 511, 513, 505, 477, 508, 517, 489, 518, 496, 528, 558, 540, 474, 501, 519, 544, 505, 529, 491, 519, 521, 498, 502, 491, 528, 493, 544, 554, 539, 497, 498, 509, 503, 539, 534, 472, 508, 513, 481, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[502, 501, 486, 513, 515, 485, 503, 534, 526, 501, 496, 519, 523, 496, 532, 484, 542, 515, 531, 483, 488, 508, 559, 509, 496, 504, 531, 531, 509, 467, 549, 493, 516, 522, 530, 510, 523, 532, 530, 543, 536, 505, 494, 534, 536, 554, 502, 525, 475, 542, 513, 516, 472, 505, 517, 465, 534, 546, 502, 484, 478, 521, 510, 465] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[514, 458, 501, 518, 541, 525, 485, 498, 463, 495, 548, 532, 536, 468, 502, 528, 544, 480, 527, 506, 493, 520, 506, 543, 508, 491, 515, 502, 497, 499, 525, 510, 481, 530, 548, 511, 480, 509, 516, 540, 542, 530, 473, 504, 482, 485, 519, 498, 521, 507, 492, 539, 535, 550, 500, 476, 532, 523, 564, 514, 519, 500, 510, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 548, 482, 506, 503, 480, 542, 541, 10, 0, 0, 0, 0, 0, 0, 0, 532, 531, 513, 544, 507, 471, 501, 487, 0, 0, 0, 0, 0, 0, 0, 0, 530, 483, 486, 496, 505, 533, 515, 540, 0, 0, 0, 0, 0, 0, 0, 0, 529, 480, 494, 489, 498, 544, 497, 445, 44, 2, 0, 0, 0, 0, 0, 0, 516, 537, 514, 517, 534, 495, 504, 485, 0, 0, 0, 0, 0, 0, 0, 0, 492, 536, 527, 522, 511, 506, 507, 504, 5, 0, 0, 0, 0, 0, 0, 0, 500, 508, 521, 509, 509, 534, 533, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 501, 500, 517, 479, 575, 472, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4086, 4088, 4022, 4102, 4110, 4118, 4090] avg=4096.0000 max=4152 max_rank=0 min=4022 min_rank=3 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 510, 497, 519, 496, 517, 518, 506, 10, 0, 0, 0, 0, 0, 0, 0, 525, 531, 489, 488, 531, 536, 473, 512, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 493, 493, 494, 534, 540, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 582, 483, 502, 545, 514, 509, 511, 40, 2, 0, 0, 0, 0, 0, 0, 533, 529, 529, 494, 552, 517, 541, 447, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 508, 503, 513, 509, 528, 518, 5, 0, 0, 0, 0, 0, 0, 0, 482, 532, 490, 522, 496, 509, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 529, 501, 534, 488, 485, 499, 457, 543, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4085, 4063, 4200, 4142, 4093, 4079, 4059] avg=4096.0000 max=4200 max_rank=3 min=4047 min_rank=0 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 534, 493, 531, 486, 502, 521, 531, 9, 0, 0, 0, 0, 0, 0, 0, 502, 513, 509, 512, 523, 533, 520, 531, 0, 0, 0, 0, 0, 0, 0, 0, 468, 533, 505, 529, 507, 541, 503, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 477, 497, 487, 524, 538, 520, 43, 2, 0, 0, 0, 0, 0, 0, 585, 527, 548, 520, 485, 513, 525, 472, 0, 0, 0, 0, 0, 0, 0, 0, 487, 536, 517, 503, 489, 473, 508, 537, 5, 0, 0, 0, 0, 0, 0, 0, 536, 537, 494, 456, 504, 509, 502, 503, 0, 0, 0, 0, 0, 0, 0, 0, 480, 530, 472, 530, 488, 519, 493, 502, 16, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4143, 4081, 4104, 4175, 4055, 4041, 4036] avg=4096.0000 max=4175 max_rank=4 min=4036 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 483, 504, 543, 502, 471, 549, 538, 9, 0, 0, 0, 0, 0, 0, 0, 465, 497, 542, 512, 528, 531, 510, 541, 0, 0, 0, 0, 0, 0, 0, 0, 512, 472, 527, 482, 539, 498, 492, 482, 0, 0, 0, 0, 0, 0, 0, 0, 497, 505, 502, 492, 471, 484, 522, 518, 44, 2, 0, 0, 0, 0, 0, 0, 491, 520, 473, 524, 500, 536, 502, 485, 0, 0, 0, 0, 0, 0, 0, 0, 490, 539, 516, 529, 504, 523, 542, 515, 5, 0, 0, 0, 0, 0, 0, 0, 539, 512, 497, 551, 536, 535, 493, 502, 0, 0, 0, 0, 0, 0, 0, 0, 552, 455, 509, 500, 563, 490, 537, 529, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4126, 4004, 4037, 4031, 4163, 4165, 4156] avg=4096.0000 max=4165 max_rank=6 min=4004 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[502, 501, 486, 513, 515, 485, 503, 534, 10, 0, 0, 0, 0, 0, 0, 0, 526, 501, 496, 514, 523, 496, 532, 484, 0, 0, 0, 0, 0, 0, 0, 0, 542, 515, 531, 483, 488, 508, 542, 509, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 531, 531, 509, 467, 549, 493, 45, 2, 0, 0, 0, 0, 0, 0, 516, 522, 530, 510, 523, 532, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 536, 505, 494, 534, 536, 554, 502, 525, 5, 0, 0, 0, 0, 0, 0, 0, 475, 524, 513, 516, 472, 505, 517, 465, 0, 0, 0, 0, 0, 0, 0, 0, 534, 546, 502, 484, 478, 521, 510, 465, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4072, 4118, 4127, 4161, 4191, 3987, 4063] avg=4096.0000 max=4191 max_rank=5 min=3987 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[486, 488, 546, 505, 502, 505, 519, 497, 10, 0, 0, 0, 0, 0, 0, 0, 507, 522, 502, 509, 497, 539, 522, 554, 0, 0, 0, 0, 0, 0, 0, 0, 542, 536, 534, 533, 557, 525, 522, 533, 0, 0, 0, 0, 0, 0, 0, 0, 515, 512, 471, 509, 519, 499, 504, 518, 43, 2, 0, 0, 0, 0, 0, 0, 507, 505, 512, 503, 508, 487, 471, 482, 0, 0, 0, 0, 0, 0, 0, 0, 472, 484, 513, 548, 523, 502, 503, 503, 5, 0, 0, 0, 0, 0, 0, 0, 515, 519, 527, 521, 507, 536, 511, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 517, 492, 471, 508, 492, 501, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4152, 4282, 4092, 3975, 4053, 4137, 4019] avg=4096.0000 max=4282 max_rank=2 min=3975 min_rank=4 imbalance_ratio=1.045410 max_load_reduction=28486 max_load_reduction_pct=0.869324 moved_to_echo=16184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 532, 551, 496, 502, 530, 499, 523, 15, 6, 0, 0, 0, 0, 0, 0, 506, 526, 496, 505, 510, 501, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 483, 514, 528, 465, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 505, 477, 508, 517, 489, 518, 48, 6, 0, 0, 0, 0, 0, 0, 496, 528, 558, 540, 474, 501, 519, 496, 0, 0, 0, 0, 0, 0, 0, 0, 505, 529, 491, 519, 521, 498, 502, 491, 9, 0, 0, 0, 0, 0, 0, 0, 528, 469, 544, 554, 539, 497, 498, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 534, 472, 508, 513, 481, 523, 18, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4194, 4037, 4028, 4092, 4112, 4065, 4138, 4102] avg=4096.0000 max=4194 max_rank=0 min=4028 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 458, 501, 518, 541, 525, 485, 498, 9, 0, 0, 0, 0, 0, 0, 0, 463, 495, 548, 527, 536, 468, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 544, 480, 527, 506, 493, 520, 491, 543, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 515, 502, 497, 499, 525, 510, 45, 2, 0, 0, 0, 0, 0, 0, 481, 530, 548, 511, 480, 509, 516, 495, 0, 0, 0, 0, 0, 0, 0, 0, 542, 530, 473, 504, 482, 485, 519, 498, 5, 0, 0, 0, 0, 0, 0, 0, 521, 490, 492, 539, 535, 550, 500, 476, 0, 0, 0, 0, 0, 0, 0, 0, 532, 523, 564, 514, 519, 500, 510, 560, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4067, 4104, 4094, 4070, 4038, 4103, 4243] avg=4096.0000 max=4243 max_rank=7 min=4038 min_rank=5 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[528, 479, 520, 549, 517, 534, 506, 510, 516, 501, 489, 522, 511, 476, 505, 500, 534, 515, 528, 504, 504, 494, 518, 534, 500, 514, 483, 532, 504, 491, 509, 501, 541, 484, 503, 496, 532, 543, 498, 521, 501, 541, 467, 525, 512, 516, 530, 498, 476, 537, 487, 502, 501, 547, 543, 542, 522, 522, 509, 484, 501, 503, 524, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[511, 506, 505, 492, 472, 472, 531, 540, 534, 489, 542, 500, 515, 473, 522, 492, 471, 556, 490, 506, 518, 534, 510, 546, 524, 484, 512, 514, 515, 506, 528, 533, 547, 507, 490, 517, 520, 519, 488, 499, 473, 517, 524, 485, 512, 515, 524, 496, 534, 513, 504, 523, 501, 530, 539, 535, 509, 503, 502, 540, 524, 496, 525, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[520, 516, 512, 484, 554, 474, 474, 530, 490, 541, 509, 503, 512, 549, 506, 493, 507, 495, 539, 516, 514, 521, 507, 482, 510, 520, 523, 507, 500, 512, 494, 490, 504, 541, 511, 484, 532, 471, 525, 496, 552, 513, 538, 494, 523, 531, 501, 541, 504, 484, 581, 490, 520, 531, 532, 464, 495, 534, 503, 505, 527, 493, 517, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[564, 536, 468, 491, 494, 503, 485, 521, 539, 498, 513, 527, 497, 506, 524, 510, 516, 514, 522, 509, 548, 509, 479, 539, 520, 522, 522, 541, 492, 505, 513, 498, 474, 515, 542, 524, 539, 500, 500, 501, 503, 531, 522, 504, 514, 509, 532, 460, 513, 490, 536, 525, 511, 523, 525, 491, 508, 542, 522, 515, 499, 478, 482, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[509, 490, 514, 541, 499, 510, 508, 487, 512, 509, 551, 496, 482, 485, 503, 547, 503, 538, 507, 505, 558, 529, 510, 519, 524, 501, 486, 511, 537, 518, 509, 478, 481, 532, 522, 510, 545, 521, 521, 511, 476, 506, 499, 548, 561, 485, 537, 483, 510, 521, 507, 537, 488, 520, 531, 496, 496, 541, 516, 515, 485, 463, 508, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[545, 533, 491, 504, 476, 499, 524, 521, 514, 496, 509, 508, 530, 515, 503, 498, 525, 538, 557, 521, 562, 458, 511, 499, 485, 533, 526, 529, 507, 501, 535, 548, 500, 527, 544, 490, 526, 461, 505, 500, 529, 529, 527, 489, 501, 482, 506, 448, 513, 448, 489, 528, 508, 556, 526, 535, 524, 507, 489, 518, 496, 520, 524, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[540, 521, 530, 493, 542, 482, 491, 533, 490, 521, 487, 505, 494, 515, 486, 479, 527, 505, 526, 519, 515, 488, 501, 542, 536, 547, 525, 505, 483, 528, 551, 480, 501, 523, 496, 496, 495, 532, 516, 554, 503, 518, 523, 538, 471, 521, 547, 520, 472, 539, 492, 533, 517, 538, 485, 497, 549, 469, 473, 498, 481, 508, 532, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[516, 543, 549, 499, 522, 523, 513, 484, 518, 518, 489, 511, 518, 483, 528, 507, 531, 471, 546, 488, 482, 489, 518, 499, 544, 535, 576, 513, 512, 542, 503, 487, 520, 498, 492, 527, 487, 480, 490, 538, 503, 510, 531, 510, 528, 496, 529, 528, 514, 526, 500, 512, 535, 512, 500, 460, 485, 548, 506, 498, 530, 507, 541, 470] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 479, 520, 549, 517, 534, 506, 510, 2, 2, 0, 0, 0, 0, 0, 0, 516, 501, 489, 522, 511, 476, 505, 500, 24, 7, 0, 0, 0, 0, 0, 0, 534, 515, 504, 504, 504, 494, 518, 534, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 483, 532, 504, 491, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 541, 484, 503, 496, 532, 543, 498, 521, 8, 0, 0, 0, 0, 0, 0, 0, 501, 541, 467, 525, 512, 516, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 476, 537, 487, 502, 501, 534, 543, 542, 0, 0, 0, 0, 0, 0, 0, 0, 522, 522, 509, 484, 501, 503, 524, 532, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4051, 4107, 4017, 4126, 4088, 4122, 4110] avg=4096.0000 max=4147 max_rank=0 min=4017 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[540, 521, 530, 493, 542, 482, 491, 533, 2, 2, 0, 0, 0, 0, 0, 0, 490, 521, 487, 505, 494, 515, 486, 479, 24, 7, 0, 0, 0, 0, 0, 0, 527, 505, 502, 519, 515, 488, 501, 542, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 525, 505, 483, 528, 551, 480, 0, 0, 0, 0, 0, 0, 0, 0, 501, 523, 496, 496, 495, 532, 516, 554, 7, 0, 0, 0, 0, 0, 0, 0, 503, 518, 523, 538, 471, 521, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 472, 539, 492, 533, 517, 526, 485, 497, 0, 0, 0, 0, 0, 0, 0, 0, 549, 469, 473, 498, 481, 508, 532, 544, 11, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4008, 4099, 4137, 4120, 4139, 4061, 4068] avg=4096.0000 max=4139 max_rank=5 min=4008 min_rank=1 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 543, 549, 499, 522, 523, 513, 484, 2, 2, 0, 0, 0, 0, 0, 0, 518, 518, 489, 511, 518, 483, 528, 507, 25, 7, 0, 0, 0, 0, 0, 0, 531, 471, 521, 488, 482, 489, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0, 544, 517, 576, 513, 512, 542, 503, 487, 0, 0, 0, 0, 0, 0, 0, 0, 520, 498, 492, 527, 487, 480, 490, 538, 7, 0, 0, 0, 0, 0, 0, 0, 503, 510, 531, 510, 528, 496, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 514, 526, 500, 512, 535, 500, 500, 460, 0, 0, 0, 0, 0, 0, 0, 0, 485, 548, 506, 498, 530, 507, 541, 470, 11, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4104, 3999, 4194, 4039, 4133, 4047, 4099] avg=4096.0000 max=4194 max_rank=3 min=3999 min_rank=2 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[511, 506, 505, 492, 472, 472, 531, 540, 2, 2, 0, 0, 0, 0, 0, 0, 534, 489, 542, 500, 515, 473, 522, 492, 23, 6, 0, 0, 0, 0, 0, 0, 471, 556, 467, 506, 518, 534, 510, 546, 0, 0, 0, 0, 0, 0, 0, 0, 524, 468, 512, 514, 515, 506, 528, 533, 0, 0, 0, 0, 0, 0, 0, 0, 547, 507, 490, 517, 520, 519, 488, 499, 7, 0, 0, 0, 0, 0, 0, 0, 473, 517, 524, 485, 512, 515, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 534, 513, 504, 523, 501, 518, 539, 535, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 502, 540, 524, 496, 525, 514, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4096, 4108, 4100, 4094, 4044, 4167, 4126] avg=4096.0000 max=4167 max_rank=6 min=4033 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 516, 512, 484, 554, 474, 474, 530, 2, 2, 0, 0, 0, 0, 0, 0, 490, 541, 509, 503, 512, 549, 506, 493, 25, 7, 0, 0, 0, 0, 0, 0, 507, 495, 514, 516, 514, 521, 507, 482, 0, 0, 0, 0, 0, 0, 0, 0, 510, 503, 523, 507, 500, 512, 494, 490, 0, 0, 0, 0, 0, 0, 0, 0, 504, 541, 511, 484, 532, 471, 525, 496, 7, 0, 0, 0, 0, 0, 0, 0, 552, 513, 538, 494, 523, 531, 499, 541, 0, 0, 0, 0, 0, 0, 0, 0, 504, 484, 581, 490, 520, 519, 532, 464, 0, 0, 0, 0, 0, 0, 0, 0, 495, 534, 503, 505, 527, 493, 517, 527, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4135, 4056, 4039, 4071, 4191, 4094, 4114] avg=4096.0000 max=4191 max_rank=5 min=4039 min_rank=3 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[545, 533, 491, 504, 476, 499, 524, 521, 2, 2, 0, 0, 0, 0, 0, 0, 514, 496, 509, 508, 530, 515, 503, 498, 26, 7, 0, 0, 0, 0, 0, 0, 525, 538, 531, 521, 562, 458, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 526, 529, 507, 501, 535, 548, 0, 0, 0, 0, 0, 0, 0, 0, 500, 527, 544, 490, 526, 461, 505, 500, 8, 0, 0, 0, 0, 0, 0, 0, 529, 529, 527, 489, 501, 482, 504, 448, 0, 0, 0, 0, 0, 0, 0, 0, 513, 448, 489, 528, 508, 543, 526, 535, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 489, 518, 496, 520, 524, 522, 11, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4106, 4145, 4146, 4061, 4009, 4090, 4114] avg=4096.0000 max=4146 max_rank=3 min=4009 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 490, 514, 541, 499, 510, 508, 487, 6, 4, 0, 0, 0, 0, 0, 0, 512, 509, 551, 496, 482, 485, 503, 547, 27, 11, 0, 0, 0, 0, 0, 0, 503, 538, 480, 505, 558, 529, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 476, 486, 511, 537, 518, 509, 478, 0, 0, 0, 0, 0, 0, 0, 0, 481, 532, 522, 510, 545, 521, 521, 511, 12, 0, 0, 0, 0, 0, 0, 0, 476, 506, 499, 548, 561, 485, 533, 483, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 507, 537, 488, 495, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 496, 541, 516, 515, 485, 463, 508, 520, 14, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4123, 4142, 4039, 4155, 4091, 4085, 4065] avg=4096.0000 max=4155 max_rank=4 min=4039 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[564, 536, 468, 491, 494, 503, 485, 521, 2, 2, 0, 0, 0, 0, 0, 0, 539, 498, 513, 527, 497, 506, 524, 510, 24, 7, 0, 0, 0, 0, 0, 0, 516, 514, 498, 509, 548, 509, 479, 539, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 522, 541, 492, 505, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 474, 515, 542, 524, 539, 500, 500, 501, 7, 0, 0, 0, 0, 0, 0, 0, 503, 531, 522, 504, 514, 509, 530, 460, 0, 0, 0, 0, 0, 0, 0, 0, 513, 490, 536, 525, 511, 511, 525, 491, 0, 0, 0, 0, 0, 0, 0, 0, 508, 542, 522, 515, 499, 478, 482, 513, 10, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4145, 4112, 4096, 4102, 4073, 4102, 4072] avg=4096.0000 max=4145 max_rank=1 min=4066 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[521, 501, 510, 527, 533, 546, 502, 489, 502, 528, 507, 504, 496, 508, 496, 499, 510, 538, 511, 531, 525, 495, 540, 513, 521, 496, 499, 514, 510, 511, 506, 523, 539, 519, 483, 489, 558, 497, 534, 473, 503, 497, 490, 511, 522, 506, 501, 518, 518, 526, 536, 531, 493, 532, 508, 511, 530, 503, 527, 473, 501, 508, 480, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[527, 523, 529, 538, 494, 467, 471, 517, 505, 486, 483, 516, 511, 529, 492, 514, 488, 563, 532, 519, 500, 519, 554, 525, 522, 490, 558, 537, 498, 465, 523, 515, 496, 522, 471, 517, 527, 542, 489, 538, 529, 527, 547, 475, 559, 550, 485, 474, 504, 544, 499, 504, 501, 511, 474, 505, 507, 479, 538, 502, 498, 508, 538, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[526, 554, 508, 530, 514, 522, 487, 500, 498, 530, 489, 498, 492, 521, 480, 544, 500, 508, 531, 539, 484, 532, 501, 495, 512, 530, 501, 507, 548, 484, 537, 494, 535, 487, 512, 487, 516, 537, 519, 514, 538, 519, 479, 520, 471, 515, 541, 524, 542, 481, 506, 497, 491, 519, 521, 518, 525, 524, 482, 485, 546, 510, 494, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[546, 485, 519, 498, 545, 570, 502, 486, 514, 543, 507, 517, 524, 476, 497, 518, 496, 492, 510, 511, 488, 514, 545, 526, 467, 506, 496, 512, 472, 505, 524, 503, 524, 488, 504, 490, 492, 479, 513, 512, 535, 501, 504, 489, 498, 504, 540, 548, 539, 499, 533, 561, 492, 498, 546, 517, 543, 568, 507, 513, 512, 504, 489, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[510, 542, 494, 513, 525, 512, 529, 486, 514, 510, 524, 532, 543, 509, 502, 507, 525, 498, 474, 535, 486, 538, 513, 511, 494, 528, 512, 526, 501, 513, 504, 506, 531, 551, 486, 504, 487, 516, 520, 498, 519, 512, 505, 534, 476, 502, 517, 472, 528, 526, 517, 477, 508, 502, 548, 507, 531, 523, 462, 529, 522, 492, 540, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[520, 515, 546, 515, 553, 513, 511, 490, 538, 510, 521, 527, 505, 487, 485, 533, 504, 517, 517, 529, 466, 553, 511, 477, 452, 492, 504, 493, 488, 494, 505, 523, 511, 502, 538, 508, 519, 514, 541, 498, 511, 493, 540, 499, 483, 532, 478, 513, 528, 517, 527, 526, 528, 550, 528, 535, 495, 491, 537, 508, 478, 503, 511, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[525, 510, 551, 503, 531, 527, 492, 516, 485, 523, 494, 501, 509, 545, 492, 484, 465, 501, 500, 497, 488, 529, 534, 521, 520, 540, 510, 522, 486, 496, 506, 502, 529, 519, 518, 487, 530, 504, 542, 527, 533, 522, 524, 512, 526, 476, 493, 518, 492, 494, 526, 518, 530, 500, 541, 502, 490, 491, 504, 518, 547, 530, 485, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[543, 500, 518, 503, 497, 555, 491, 519, 485, 507, 536, 529, 508, 507, 496, 508, 506, 541, 549, 460, 500, 500, 556, 521, 547, 520, 523, 474, 481, 533, 518, 473, 536, 507, 532, 502, 521, 523, 504, 508, 528, 507, 512, 500, 559, 503, 533, 508, 509, 488, 504, 520, 501, 535, 519, 478, 484, 507, 490, 532, 522, 497, 486, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 501, 510, 527, 533, 546, 502, 489, 0, 0, 0, 0, 0, 0, 0, 0, 502, 528, 507, 504, 496, 508, 496, 499, 21, 1, 0, 0, 0, 0, 0, 0, 510, 538, 511, 531, 525, 495, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 499, 514, 510, 511, 506, 523, 33, 2, 0, 0, 0, 0, 0, 0, 533, 519, 483, 489, 558, 497, 534, 473, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 490, 511, 522, 506, 501, 518, 1, 0, 0, 0, 0, 0, 0, 0, 518, 526, 536, 531, 493, 532, 485, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 527, 473, 501, 508, 480, 539, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4062, 4153, 4115, 4086, 4049, 4132, 4075] avg=4096.0000 max=4153 max_rank=2 min=4049 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[491, 510, 551, 503, 531, 527, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 485, 523, 494, 501, 509, 545, 492, 484, 22, 1, 0, 0, 0, 0, 0, 0, 465, 501, 500, 497, 488, 529, 524, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 540, 510, 522, 486, 496, 506, 502, 34, 2, 0, 0, 0, 0, 0, 0, 523, 519, 518, 487, 530, 504, 542, 527, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 524, 512, 526, 476, 493, 518, 1, 0, 0, 0, 0, 0, 0, 0, 492, 494, 526, 518, 530, 500, 517, 502, 0, 0, 0, 0, 0, 0, 0, 0, 490, 491, 504, 518, 547, 530, 485, 535, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4056, 4025, 4118, 4150, 4105, 4079, 4114] avg=4096.0000 max=4150 max_rank=4 min=4025 min_rank=2 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 485, 519, 498, 545, 570, 502, 486, 0, 0, 0, 0, 0, 0, 0, 0, 514, 543, 507, 517, 524, 476, 497, 518, 22, 1, 0, 0, 0, 0, 0, 0, 496, 492, 510, 511, 488, 514, 535, 526, 0, 0, 0, 0, 0, 0, 0, 0, 467, 506, 496, 512, 472, 505, 524, 503, 35, 2, 0, 0, 0, 0, 0, 0, 520, 488, 504, 490, 492, 479, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 535, 501, 504, 489, 498, 504, 540, 548, 0, 0, 0, 0, 0, 0, 0, 0, 539, 499, 533, 561, 492, 498, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 543, 568, 507, 513, 512, 504, 489, 512, 9, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4119, 4072, 4022, 3998, 4119, 4161, 4161] avg=4096.0000 max=4161 max_rank=6 min=3998 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 554, 508, 530, 514, 522, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 489, 498, 492, 521, 480, 544, 21, 1, 0, 0, 0, 0, 0, 0, 500, 508, 531, 539, 484, 532, 491, 495, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 501, 507, 548, 484, 537, 494, 34, 2, 0, 0, 0, 0, 0, 0, 529, 487, 512, 487, 516, 537, 519, 514, 0, 0, 0, 0, 0, 0, 0, 0, 538, 519, 479, 520, 471, 515, 541, 524, 1, 0, 0, 0, 0, 0, 0, 0, 542, 481, 506, 497, 491, 519, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 524, 482, 485, 546, 510, 494, 517, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4074, 4080, 4149, 4101, 4108, 4052, 4097] avg=4096.0000 max=4149 max_rank=3 min=4052 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[493, 523, 529, 538, 494, 467, 471, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 486, 483, 516, 511, 529, 492, 514, 19, 1, 0, 0, 0, 0, 0, 0, 488, 563, 532, 519, 500, 519, 543, 525, 0, 0, 0, 0, 0, 0, 0, 0, 522, 490, 558, 537, 498, 465, 523, 515, 34, 2, 0, 0, 0, 0, 0, 0, 492, 522, 471, 517, 527, 542, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 529, 527, 547, 475, 559, 550, 485, 474, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 499, 504, 501, 511, 453, 505, 0, 0, 0, 0, 0, 0, 0, 0, 507, 479, 538, 502, 498, 508, 538, 498, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4056, 4189, 4144, 4098, 4146, 4021, 4082] avg=4096.0000 max=4189 max_rank=2 min=4021 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[477, 542, 494, 513, 525, 512, 529, 486, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 524, 532, 543, 509, 502, 507, 23, 1, 0, 0, 0, 0, 0, 0, 525, 498, 474, 535, 486, 538, 503, 511, 0, 0, 0, 0, 0, 0, 0, 0, 494, 528, 512, 526, 501, 513, 504, 506, 33, 2, 0, 0, 0, 0, 0, 0, 525, 551, 486, 504, 487, 516, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 505, 534, 476, 502, 517, 472, 1, 0, 0, 0, 0, 0, 0, 0, 528, 526, 517, 477, 508, 502, 523, 507, 0, 0, 0, 0, 0, 0, 0, 0, 531, 523, 462, 529, 522, 492, 540, 510, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4165, 4070, 4119, 4087, 4038, 4088, 4123] avg=4096.0000 max=4165 max_rank=1 min=4038 min_rank=5 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 515, 546, 515, 553, 513, 511, 490, 0, 0, 0, 0, 0, 0, 0, 0, 538, 510, 521, 527, 505, 487, 485, 533, 27, 5, 0, 0, 0, 0, 0, 0, 504, 517, 517, 529, 466, 553, 494, 477, 0, 0, 0, 0, 0, 0, 0, 0, 452, 492, 504, 493, 488, 494, 505, 523, 36, 8, 0, 0, 0, 0, 0, 0, 501, 502, 538, 508, 519, 514, 541, 498, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 540, 499, 483, 532, 478, 513, 3, 0, 0, 0, 0, 0, 0, 0, 528, 517, 527, 526, 528, 550, 493, 535, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 537, 508, 478, 503, 511, 532, 12, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4138, 4057, 3995, 4121, 4052, 4204, 4074] avg=4096.0000 max=4204 max_rank=6 min=3995 min_rank=3 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[508, 500, 518, 503, 497, 555, 491, 519, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 536, 529, 508, 507, 496, 508, 21, 1, 0, 0, 0, 0, 0, 0, 506, 541, 549, 460, 500, 500, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0, 547, 520, 523, 474, 481, 533, 518, 473, 35, 2, 0, 0, 0, 0, 0, 0, 530, 507, 532, 502, 521, 523, 504, 508, 0, 0, 0, 0, 0, 0, 0, 0, 528, 507, 512, 500, 559, 503, 533, 508, 1, 0, 0, 0, 0, 0, 0, 0, 509, 488, 504, 520, 501, 535, 496, 478, 0, 0, 0, 0, 0, 0, 0, 0, 484, 507, 490, 532, 522, 497, 486, 509, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4098, 4122, 4106, 4127, 4151, 4031, 4042] avg=4096.0000 max=4151 max_rank=5 min=4031 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +Number of parameters in transformer block in billions: 20.11 +Number of parameters in embedding layers in billions: 0.36 +Total number of parameters in billions: 20.47 +Number of parameters in most loaded shard in billions: 20.4724 +Theoretical memory footprints: weight and optimizer=146430.14 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 68662.10546875 | max allocated: 75502.8310546875 | reserved: 79256.0 | max reserved: 79256.0 + [2026-08-26 14:57:41] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 37899.4 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097996E+01 | loss scale: 1.0 | grad norm: 29.758 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[504, 487, 500, 519, 529, 535, 512, 491, 518, 529, 484, 522, 519, 508, 533, 478, 480, 519, 517, 528, 469, 524, 529, 522, 511, 464, 522, 506, 510, 538, 527, 512, 508, 480, 528, 487, 509, 482, 515, 515, 534, 501, 504, 505, 500, 493, 528, 523, 504, 507, 533, 522, 510, 533, 511, 512, 515, 489, 525, 514, 510, 530, 547, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[513, 536, 530, 495, 519, 505, 521, 577, 509, 522, 484, 535, 511, 531, 528, 500, 479, 556, 483, 511, 514, 545, 529, 509, 515, 466, 526, 526, 476, 508, 499, 503, 511, 536, 510, 491, 505, 531, 544, 511, 520, 501, 501, 495, 522, 499, 515, 512, 484, 508, 546, 527, 488, 479, 495, 521, 533, 493, 501, 477, 490, 533, 511, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[499, 536, 522, 508, 557, 507, 543, 501, 521, 497, 536, 495, 477, 489, 477, 521, 511, 483, 535, 494, 526, 501, 501, 497, 506, 485, 505, 537, 524, 541, 473, 511, 509, 476, 546, 501, 527, 535, 520, 505, 525, 482, 520, 534, 533, 487, 518, 514, 529, 518, 524, 523, 500, 507, 550, 507, 493, 546, 462, 513, 486, 543, 517, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[553, 498, 504, 496, 503, 494, 466, 514, 518, 490, 493, 542, 538, 524, 541, 510, 541, 556, 479, 506, 531, 502, 550, 515, 459, 531, 524, 521, 503, 528, 518, 528, 510, 486, 547, 499, 518, 529, 560, 527, 548, 524, 529, 489, 501, 495, 505, 528, 507, 508, 531, 496, 494, 495, 494, 498, 499, 480, 491, 525, 496, 510, 493, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[513, 536, 512, 511, 514, 512, 479, 507, 502, 542, 504, 520, 491, 515, 535, 511, 510, 520, 505, 510, 473, 494, 549, 519, 520, 538, 569, 507, 542, 505, 484, 497, 533, 506, 542, 533, 507, 494, 514, 502, 502, 541, 509, 481, 489, 491, 515, 502, 512, 535, 504, 496, 505, 510, 518, 553, 483, 501, 513, 505, 503, 496, 510, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[532, 557, 488, 487, 526, 536, 529, 521, 553, 479, 521, 510, 519, 519, 533, 520, 518, 476, 496, 516, 517, 525, 515, 489, 482, 528, 539, 541, 483, 533, 496, 502, 496, 504, 477, 518, 513, 528, 511, 525, 499, 518, 518, 493, 496, 530, 530, 557, 511, 514, 505, 556, 520, 528, 486, 486, 483, 482, 497, 460, 503, 535, 517, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[485, 507, 536, 543, 525, 499, 508, 488, 522, 512, 518, 507, 554, 517, 499, 527, 519, 513, 545, 518, 541, 503, 554, 470, 507, 505, 520, 506, 546, 509, 468, 537, 510, 496, 476, 509, 488, 537, 518, 515, 493, 519, 505, 509, 504, 556, 486, 483, 525, 510, 509, 536, 508, 524, 497, 491, 484, 526, 490, 507, 511, 522, 487, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[510, 513, 475, 524, 549, 558, 487, 516, 499, 533, 549, 512, 514, 479, 512, 518, 467, 545, 543, 497, 526, 487, 508, 503, 487, 466, 503, 517, 502, 495, 512, 500, 504, 538, 530, 516, 482, 536, 510, 514, 509, 506, 503, 526, 513, 478, 513, 507, 575, 531, 490, 501, 531, 536, 520, 490, 489, 459, 542, 520, 501, 584, 510, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 487, 500, 519, 506, 535, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 484, 522, 519, 508, 513, 478, 0, 0, 0, 0, 0, 0, 0, 0, 480, 519, 517, 528, 469, 524, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 511, 464, 522, 506, 510, 538, 527, 512, 13, 1, 0, 0, 0, 0, 0, 0, 508, 480, 528, 487, 509, 482, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 504, 505, 500, 493, 528, 523, 7, 5, 0, 0, 0, 0, 0, 0, 504, 507, 533, 509, 510, 533, 511, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 489, 525, 514, 510, 530, 547, 548, 23, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4071, 4083, 4104, 4016, 4100, 4119, 4221] avg=4096.0000 max=4221 max_rank=7 min=4016 min_rank=4 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 536, 530, 495, 496, 505, 521, 577, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 484, 535, 511, 531, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 479, 556, 483, 511, 514, 545, 524, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 466, 526, 526, 476, 508, 499, 503, 13, 1, 0, 0, 0, 0, 0, 0, 511, 536, 510, 491, 505, 531, 536, 511, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 501, 495, 522, 499, 515, 512, 7, 5, 0, 0, 0, 0, 0, 0, 484, 508, 546, 514, 488, 479, 495, 521, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 501, 477, 490, 533, 511, 517, 23, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4100, 4121, 4033, 4131, 4077, 4035, 4098] avg=4096.0000 max=4173 max_rank=0 min=4033 min_rank=3 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 536, 512, 511, 491, 512, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 502, 542, 504, 520, 491, 515, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 510, 520, 505, 510, 473, 494, 544, 519, 0, 0, 0, 0, 0, 0, 0, 0, 520, 538, 569, 507, 542, 505, 484, 497, 12, 1, 0, 0, 0, 0, 0, 0, 533, 506, 542, 533, 507, 494, 507, 502, 0, 0, 0, 0, 0, 0, 0, 0, 502, 541, 509, 481, 489, 491, 515, 502, 6, 5, 0, 0, 0, 0, 0, 0, 512, 535, 504, 484, 505, 510, 518, 553, 0, 0, 0, 0, 0, 0, 0, 0, 483, 501, 513, 505, 503, 496, 510, 517, 23, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4100, 4075, 4175, 4124, 4041, 4121, 4071] avg=4096.0000 max=4175 max_rank=3 min=4041 min_rank=5 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 536, 522, 508, 533, 507, 543, 501, 0, 0, 0, 0, 0, 0, 0, 0, 521, 497, 536, 495, 477, 489, 459, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 483, 535, 494, 526, 501, 496, 497, 0, 0, 0, 0, 0, 0, 0, 0, 506, 485, 505, 537, 524, 541, 473, 511, 13, 1, 0, 0, 0, 0, 0, 0, 509, 476, 546, 501, 527, 535, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 482, 520, 534, 533, 487, 518, 514, 7, 5, 0, 0, 0, 0, 0, 0, 529, 518, 524, 510, 500, 507, 550, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 546, 462, 513, 486, 543, 517, 502, 24, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 3995, 4043, 4096, 4111, 4125, 4145, 4104] avg=4096.0000 max=4149 max_rank=0 min=3995 min_rank=1 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[553, 498, 504, 496, 477, 494, 466, 514, 0, 0, 0, 0, 0, 0, 0, 0, 518, 490, 493, 542, 538, 524, 518, 510, 0, 0, 0, 0, 0, 0, 0, 0, 541, 556, 479, 506, 531, 502, 540, 515, 0, 0, 0, 0, 0, 0, 0, 0, 459, 531, 524, 521, 503, 528, 518, 528, 15, 8, 0, 0, 0, 0, 0, 0, 510, 486, 547, 499, 518, 529, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 548, 524, 529, 489, 501, 495, 505, 528, 11, 10, 0, 0, 0, 0, 0, 0, 507, 508, 531, 479, 494, 495, 494, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 480, 491, 525, 496, 510, 493, 480, 26, 23, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4133, 4170, 4135, 4157, 4140, 4006, 4025] avg=4096.0000 max=4170 max_rank=2 min=4002 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[532, 557, 488, 487, 503, 536, 529, 521, 0, 0, 0, 0, 0, 0, 0, 0, 553, 479, 521, 510, 519, 519, 513, 520, 0, 0, 0, 0, 0, 0, 0, 0, 518, 476, 496, 516, 517, 525, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 482, 528, 539, 541, 483, 533, 496, 502, 13, 1, 0, 0, 0, 0, 0, 0, 496, 504, 477, 518, 513, 528, 504, 525, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 518, 493, 496, 530, 530, 557, 6, 5, 0, 0, 0, 0, 0, 0, 511, 514, 505, 543, 520, 528, 486, 486, 0, 0, 0, 0, 0, 0, 0, 0, 483, 482, 497, 460, 503, 535, 517, 486, 23, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4134, 4047, 4118, 4065, 4152, 4093, 4006] avg=4096.0000 max=4153 max_rank=0 min=4006 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[485, 507, 536, 543, 502, 499, 508, 488, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 518, 507, 554, 517, 481, 527, 0, 0, 0, 0, 0, 0, 0, 0, 519, 513, 545, 518, 541, 503, 549, 470, 0, 0, 0, 0, 0, 0, 0, 0, 507, 505, 520, 506, 546, 509, 468, 537, 13, 1, 0, 0, 0, 0, 0, 0, 510, 496, 476, 509, 488, 537, 510, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 519, 505, 509, 504, 556, 486, 483, 7, 5, 0, 0, 0, 0, 0, 0, 525, 510, 509, 523, 508, 524, 497, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 526, 490, 507, 511, 522, 487, 529, 23, 18, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4138, 4158, 4112, 4041, 4067, 4087, 4097] avg=4096.0000 max=4158 max_rank=2 min=4041 min_rank=4 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 513, 475, 524, 525, 558, 487, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 533, 549, 512, 514, 479, 493, 518, 0, 0, 0, 0, 0, 0, 0, 0, 467, 545, 543, 497, 526, 487, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 487, 466, 503, 517, 502, 495, 512, 500, 12, 1, 0, 0, 0, 0, 0, 0, 504, 538, 530, 516, 482, 536, 503, 514, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 503, 526, 513, 478, 513, 507, 6, 5, 0, 0, 0, 0, 0, 0, 575, 531, 490, 489, 531, 536, 520, 490, 0, 0, 0, 0, 0, 0, 0, 0, 489, 459, 542, 520, 501, 584, 510, 498, 24, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4097, 4071, 3995, 4123, 4066, 4162, 4146] avg=4096.0000 max=4162 max_rank=6 min=3995 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16497 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4352, 4096, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[509, 528, 551, 515, 528, 486, 555, 509, 537, 497, 503, 504, 527, 525, 510, 490, 503, 513, 527, 535, 511, 539, 490, 466, 492, 522, 540, 512, 558, 482, 510, 473, 495, 496, 530, 496, 519, 510, 544, 505, 516, 507, 540, 521, 507, 505, 527, 475, 501, 524, 493, 533, 538, 484, 511, 481, 516, 500, 550, 504, 490, 519, 504, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[489, 528, 508, 481, 482, 506, 547, 508, 510, 479, 493, 514, 533, 571, 512, 532, 498, 492, 517, 498, 520, 520, 516, 516, 505, 524, 480, 492, 536, 498, 518, 521, 497, 485, 519, 531, 497, 515, 515, 514, 513, 486, 525, 540, 487, 522, 538, 529, 468, 511, 558, 536, 494, 511, 481, 517, 531, 524, 490, 518, 482, 545, 514, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[487, 540, 453, 549, 500, 513, 518, 498, 474, 489, 504, 510, 543, 513, 510, 513, 497, 504, 506, 487, 508, 556, 483, 501, 482, 506, 509, 495, 531, 540, 517, 533, 526, 515, 573, 519, 542, 469, 493, 497, 537, 530, 540, 522, 506, 525, 517, 528, 555, 519, 499, 465, 514, 498, 511, 499, 527, 494, 563, 463, 482, 501, 545, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[517, 513, 538, 508, 503, 480, 490, 524, 492, 517, 521, 541, 541, 526, 484, 548, 519, 546, 526, 535, 526, 511, 516, 532, 473, 519, 497, 497, 487, 454, 550, 529, 500, 524, 505, 502, 510, 522, 514, 492, 510, 506, 509, 535, 502, 509, 545, 519, 495, 489, 480, 522, 481, 491, 516, 508, 508, 493, 541, 537, 472, 516, 536, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[516, 541, 497, 489, 486, 528, 551, 540, 541, 511, 495, 549, 509, 549, 521, 539, 542, 513, 524, 521, 509, 507, 498, 519, 493, 517, 530, 519, 502, 490, 511, 483, 470, 538, 503, 500, 505, 532, 485, 490, 511, 536, 508, 488, 534, 482, 508, 524, 502, 516, 504, 501, 521, 497, 500, 488, 532, 493, 530, 501, 524, 500, 488, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[493, 519, 501, 537, 510, 485, 493, 521, 523, 536, 508, 566, 506, 487, 466, 503, 514, 504, 486, 500, 520, 521, 506, 532, 525, 527, 490, 506, 533, 522, 511, 519, 501, 507, 507, 518, 491, 525, 505, 522, 529, 478, 502, 532, 511, 521, 538, 511, 535, 514, 516, 545, 493, 512, 526, 515, 499, 489, 493, 514, 518, 506, 507, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[506, 569, 502, 532, 535, 495, 508, 490, 576, 515, 516, 464, 551, 518, 530, 496, 559, 493, 501, 517, 493, 545, 523, 500, 506, 532, 502, 514, 504, 489, 500, 484, 528, 520, 522, 527, 483, 513, 522, 526, 470, 508, 518, 535, 519, 513, 474, 486, 512, 499, 499, 503, 539, 501, 531, 455, 529, 527, 522, 509, 491, 496, 513, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[546, 512, 535, 500, 572, 513, 538, 519, 463, 507, 530, 515, 566, 490, 525, 494, 532, 503, 519, 516, 506, 549, 486, 471, 528, 522, 522, 518, 537, 480, 477, 510, 484, 483, 539, 506, 492, 508, 517, 552, 503, 487, 497, 534, 500, 530, 526, 496, 528, 501, 536, 520, 508, 503, 501, 563, 511, 527, 475, 492, 509, 462, 493, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 508, 551, 515, 528, 486, 555, 509, 0, 0, 0, 0, 0, 0, 0, 0, 537, 497, 503, 504, 496, 525, 510, 490, 0, 0, 0, 0, 0, 0, 0, 0, 503, 513, 527, 535, 511, 530, 490, 466, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 540, 512, 558, 482, 510, 473, 15, 7, 0, 0, 0, 0, 0, 0, 495, 496, 530, 496, 519, 510, 544, 505, 9, 0, 0, 0, 0, 0, 0, 0, 516, 507, 540, 507, 507, 505, 527, 475, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 493, 533, 538, 484, 511, 481, 24, 0, 0, 0, 0, 0, 0, 0, 516, 500, 550, 504, 490, 519, 504, 480, 14, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4062, 4075, 4111, 4104, 4084, 4089, 4082] avg=4096.0000 max=4161 max_rank=0 min=4062 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[487, 519, 453, 549, 500, 513, 518, 498, 0, 0, 0, 0, 0, 0, 0, 0, 474, 489, 504, 510, 511, 513, 510, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 504, 506, 487, 508, 547, 483, 501, 0, 0, 0, 0, 0, 0, 0, 0, 482, 506, 509, 495, 531, 540, 517, 533, 15, 7, 0, 0, 0, 0, 0, 0, 526, 515, 573, 519, 542, 469, 493, 497, 9, 0, 0, 0, 0, 0, 0, 0, 537, 530, 540, 508, 506, 525, 517, 528, 0, 0, 0, 0, 0, 0, 0, 0, 555, 519, 499, 465, 514, 498, 511, 499, 25, 0, 0, 0, 0, 0, 0, 0, 527, 494, 563, 463, 482, 501, 545, 525, 14, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4024, 4033, 4135, 4143, 4191, 4085, 4120] avg=4096.0000 max=4191 max_rank=5 min=4024 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 494, 538, 508, 503, 480, 490, 524, 0, 0, 0, 0, 0, 0, 0, 0, 492, 517, 521, 541, 509, 526, 484, 548, 0, 0, 0, 0, 0, 0, 0, 0, 519, 546, 526, 535, 526, 502, 516, 532, 0, 0, 0, 0, 0, 0, 0, 0, 473, 519, 497, 497, 487, 454, 550, 529, 14, 7, 0, 0, 0, 0, 0, 0, 500, 524, 505, 502, 510, 522, 514, 492, 9, 0, 0, 0, 0, 0, 0, 0, 510, 506, 509, 520, 502, 509, 545, 519, 0, 0, 0, 0, 0, 0, 0, 0, 495, 489, 480, 522, 481, 491, 516, 508, 25, 0, 0, 0, 0, 0, 0, 0, 508, 493, 541, 537, 472, 516, 536, 519, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4138, 4202, 4027, 4078, 4120, 4007, 4142] avg=4096.0000 max=4202 max_rank=2 min=4007 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 508, 508, 481, 482, 506, 547, 508, 0, 0, 0, 0, 0, 0, 0, 0, 510, 479, 493, 514, 501, 571, 512, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 492, 517, 498, 520, 511, 516, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 480, 492, 536, 498, 518, 521, 15, 7, 0, 0, 0, 0, 0, 0, 497, 485, 519, 531, 497, 515, 515, 514, 9, 0, 0, 0, 0, 0, 0, 0, 513, 486, 525, 525, 487, 522, 538, 529, 0, 0, 0, 0, 0, 0, 0, 0, 468, 511, 558, 536, 494, 511, 481, 517, 25, 0, 0, 0, 0, 0, 0, 0, 531, 524, 490, 518, 482, 545, 514, 531, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4112, 4068, 4096, 4082, 4125, 4101, 4155] avg=4096.0000 max=4155 max_rank=7 min=4029 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[546, 493, 535, 500, 572, 513, 538, 519, 0, 0, 0, 0, 0, 0, 0, 0, 463, 507, 530, 515, 533, 490, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 532, 503, 519, 516, 506, 540, 486, 471, 0, 0, 0, 0, 0, 0, 0, 0, 528, 522, 522, 518, 537, 480, 477, 510, 14, 7, 0, 0, 0, 0, 0, 0, 484, 483, 539, 506, 492, 508, 517, 552, 9, 0, 0, 0, 0, 0, 0, 0, 503, 487, 497, 520, 500, 530, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 528, 501, 536, 520, 508, 503, 501, 563, 26, 0, 0, 0, 0, 0, 0, 0, 511, 527, 475, 492, 509, 462, 493, 484, 14, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4216, 4057, 4073, 4115, 4090, 4059, 4186, 3972] avg=4096.0000 max=4216 max_rank=0 min=3972 min_rank=7 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 499, 501, 537, 510, 485, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 536, 508, 566, 477, 487, 466, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 504, 486, 500, 520, 512, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 525, 527, 490, 506, 533, 522, 511, 519, 15, 6, 0, 0, 0, 0, 0, 0, 501, 507, 507, 518, 491, 525, 505, 522, 9, 0, 0, 0, 0, 0, 0, 0, 529, 478, 502, 518, 511, 521, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 516, 545, 493, 512, 526, 515, 23, 0, 0, 0, 0, 0, 0, 0, 499, 489, 493, 514, 518, 506, 507, 518, 14, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4066, 4074, 4154, 4085, 4108, 4179, 4063] avg=4096.0000 max=4179 max_rank=6 min=4039 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 520, 497, 489, 486, 528, 551, 540, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 495, 549, 479, 549, 521, 539, 0, 0, 0, 0, 0, 0, 0, 0, 542, 513, 524, 521, 509, 499, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 530, 519, 502, 490, 511, 483, 15, 7, 0, 0, 0, 0, 0, 0, 470, 538, 503, 500, 505, 532, 485, 490, 8, 0, 0, 0, 0, 0, 0, 0, 511, 536, 508, 475, 534, 482, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 504, 501, 521, 497, 500, 488, 23, 0, 0, 0, 0, 0, 0, 0, 532, 493, 530, 501, 524, 500, 488, 517, 13, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4184, 4125, 4067, 4031, 4078, 4052, 4104] avg=4096.0000 max=4184 max_rank=1 min=4031 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[506, 537, 502, 532, 535, 495, 508, 490, 0, 0, 0, 0, 0, 0, 0, 0, 576, 515, 516, 464, 510, 518, 530, 496, 0, 0, 0, 0, 0, 0, 0, 0, 559, 493, 501, 517, 493, 532, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 506, 532, 502, 514, 504, 489, 500, 484, 21, 11, 0, 0, 0, 0, 0, 0, 528, 520, 522, 527, 483, 513, 522, 526, 13, 1, 0, 0, 0, 0, 0, 0, 470, 508, 518, 515, 519, 513, 474, 486, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 499, 503, 539, 501, 531, 455, 30, 0, 0, 0, 0, 0, 0, 0, 529, 527, 522, 509, 491, 496, 513, 513, 19, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4125, 4118, 4063, 4155, 4003, 4069, 4130] avg=4096.0000 max=4155 max_rank=4 min=4003 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 3968, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[495, 488, 487, 530, 519, 517, 522, 527, 484, 482, 517, 544, 526, 514, 488, 506, 501, 486, 475, 549, 531, 523, 519, 510, 492, 537, 502, 463, 522, 525, 518, 522, 482, 531, 529, 520, 536, 513, 511, 490, 523, 519, 514, 517, 523, 496, 534, 501, 535, 515, 520, 510, 511, 516, 525, 524, 506, 506, 514, 533, 482, 500, 503, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[530, 535, 514, 520, 515, 495, 495, 487, 504, 545, 503, 553, 523, 526, 481, 539, 522, 523, 506, 521, 508, 494, 527, 498, 521, 510, 512, 528, 528, 501, 558, 518, 463, 506, 478, 531, 519, 521, 532, 510, 515, 508, 526, 525, 498, 541, 520, 487, 480, 529, 494, 483, 529, 529, 521, 461, 510, 465, 519, 522, 490, 512, 495, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[505, 515, 478, 492, 531, 490, 519, 501, 515, 525, 520, 495, 507, 539, 505, 507, 509, 458, 493, 529, 514, 519, 522, 516, 536, 538, 530, 517, 512, 476, 522, 501, 599, 512, 514, 525, 478, 503, 485, 546, 537, 496, 516, 519, 528, 511, 491, 523, 489, 492, 517, 533, 477, 524, 494, 478, 497, 559, 530, 481, 544, 516, 516, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[539, 499, 491, 576, 545, 537, 520, 507, 546, 540, 523, 512, 528, 548, 506, 503, 526, 474, 527, 549, 509, 519, 550, 550, 502, 493, 524, 458, 541, 486, 505, 497, 526, 498, 517, 524, 494, 485, 512, 476, 523, 519, 514, 461, 489, 530, 484, 495, 491, 495, 538, 478, 495, 515, 515, 502, 467, 534, 526, 506, 508, 491, 528, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[504, 536, 538, 533, 504, 517, 529, 522, 521, 519, 534, 516, 506, 547, 496, 499, 505, 501, 525, 517, 543, 506, 507, 500, 467, 474, 535, 501, 518, 522, 540, 508, 521, 517, 544, 473, 516, 526, 507, 526, 501, 527, 474, 532, 515, 492, 492, 464, 515, 517, 494, 569, 509, 490, 521, 522, 498, 499, 475, 523, 534, 485, 520, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[483, 470, 503, 495, 519, 512, 486, 483, 483, 519, 533, 522, 552, 472, 513, 518, 515, 527, 494, 523, 515, 505, 497, 533, 506, 520, 524, 519, 523, 527, 562, 460, 518, 530, 521, 493, 518, 532, 482, 512, 506, 524, 531, 492, 556, 514, 538, 523, 506, 515, 522, 483, 525, 500, 498, 492, 483, 502, 537, 519, 504, 523, 541, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[495, 501, 537, 481, 517, 497, 535, 511, 518, 531, 532, 489, 519, 510, 543, 529, 506, 521, 505, 525, 498, 476, 522, 515, 523, 518, 516, 529, 505, 491, 483, 519, 532, 511, 502, 484, 524, 529, 479, 500, 520, 516, 522, 532, 543, 508, 500, 502, 516, 509, 538, 500, 536, 478, 486, 518, 498, 508, 523, 546, 484, 493, 521, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[504, 540, 513, 582, 522, 454, 524, 490, 514, 494, 533, 538, 512, 527, 510, 505, 550, 513, 494, 524, 504, 540, 534, 501, 514, 505, 469, 532, 539, 497, 514, 519, 498, 509, 513, 481, 527, 543, 526, 500, 495, 523, 513, 520, 505, 501, 520, 495, 508, 499, 499, 529, 493, 527, 514, 507, 518, 475, 509, 485, 521, 508, 486, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 488, 487, 530, 519, 517, 522, 527, 1, 0, 0, 0, 0, 0, 0, 0, 484, 482, 477, 544, 526, 514, 488, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 486, 475, 533, 531, 523, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 492, 537, 502, 463, 522, 525, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 482, 531, 529, 520, 536, 513, 511, 490, 0, 0, 0, 0, 0, 0, 0, 0, 523, 519, 514, 517, 522, 496, 534, 501, 0, 0, 0, 0, 0, 0, 0, 0, 535, 515, 520, 510, 511, 516, 525, 524, 16, 10, 0, 0, 0, 0, 0, 0, 506, 506, 514, 533, 482, 500, 503, 508, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4021, 4078, 4081, 4112, 4126, 4182, 4082] avg=4096.0000 max=4182 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[530, 535, 514, 520, 515, 495, 495, 487, 1, 0, 0, 0, 0, 0, 0, 0, 504, 545, 463, 553, 523, 526, 481, 539, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 506, 506, 508, 494, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 521, 510, 512, 528, 528, 501, 558, 518, 0, 0, 0, 0, 0, 0, 0, 0, 463, 506, 478, 531, 519, 521, 532, 510, 0, 0, 0, 0, 0, 0, 0, 0, 515, 508, 526, 525, 497, 541, 520, 487, 0, 0, 0, 0, 0, 0, 0, 0, 480, 529, 494, 483, 529, 529, 521, 461, 15, 10, 0, 0, 0, 0, 0, 0, 510, 465, 519, 522, 490, 512, 495, 509, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4134, 4084, 4176, 4060, 4119, 4051, 4052] avg=4096.0000 max=4176 max_rank=3 min=4051 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[505, 515, 478, 492, 531, 490, 519, 501, 1, 0, 0, 0, 0, 0, 0, 0, 515, 525, 478, 495, 507, 539, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 458, 493, 513, 514, 519, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 536, 538, 530, 517, 512, 476, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 599, 512, 514, 525, 478, 503, 485, 546, 0, 0, 0, 0, 0, 0, 0, 0, 537, 496, 516, 519, 527, 511, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 489, 492, 517, 533, 477, 524, 494, 478, 16, 11, 0, 0, 0, 0, 0, 0, 497, 559, 530, 481, 544, 516, 516, 502, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4071, 4044, 4132, 4162, 4120, 4031, 4176] avg=4096.0000 max=4176 max_rank=7 min=4031 min_rank=6 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 501, 537, 481, 517, 497, 535, 511, 1, 0, 0, 0, 0, 0, 0, 0, 518, 531, 490, 489, 519, 510, 543, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 521, 505, 510, 498, 476, 522, 515, 0, 0, 0, 0, 0, 0, 0, 0, 523, 518, 516, 529, 505, 491, 483, 519, 0, 0, 0, 0, 0, 0, 0, 0, 532, 511, 502, 484, 524, 529, 479, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 516, 522, 532, 542, 508, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 538, 500, 536, 478, 486, 518, 15, 11, 0, 0, 0, 0, 0, 0, 498, 508, 523, 546, 484, 493, 521, 513, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4129, 4053, 4084, 4061, 4142, 4107, 4117] avg=4096.0000 max=4142 max_rank=5 min=4053 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[539, 499, 491, 576, 545, 537, 520, 507, 1, 0, 0, 0, 0, 0, 0, 0, 546, 540, 481, 512, 528, 548, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 526, 474, 527, 533, 509, 519, 550, 550, 0, 0, 0, 0, 0, 0, 0, 0, 502, 493, 524, 458, 541, 486, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 526, 498, 517, 524, 494, 485, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 523, 519, 514, 461, 488, 530, 484, 495, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 538, 478, 495, 515, 515, 502, 16, 11, 0, 0, 0, 0, 0, 0, 467, 534, 526, 506, 508, 491, 528, 502, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4215, 4164, 4188, 4006, 4032, 4014, 4056, 4093] avg=4096.0000 max=4215 max_rank=0 min=4006 min_rank=3 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16195 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 540, 513, 582, 522, 454, 524, 490, 1, 0, 0, 0, 0, 0, 0, 0, 514, 494, 491, 538, 512, 527, 510, 505, 0, 0, 0, 0, 0, 0, 0, 0, 550, 513, 494, 509, 504, 540, 534, 501, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 469, 532, 539, 497, 514, 519, 0, 0, 0, 0, 0, 0, 0, 0, 498, 509, 513, 481, 527, 543, 526, 500, 0, 0, 0, 0, 0, 0, 0, 0, 495, 523, 513, 520, 504, 501, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 499, 529, 493, 527, 514, 507, 15, 11, 0, 0, 0, 0, 0, 0, 518, 475, 509, 485, 521, 508, 486, 510, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4091, 4145, 4089, 4097, 4071, 4102, 4043] avg=4096.0000 max=4145 max_rank=2 min=4043 min_rank=7 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[504, 536, 538, 533, 504, 517, 529, 522, 2, 1, 0, 0, 0, 0, 0, 0, 521, 519, 484, 516, 506, 547, 496, 499, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 525, 495, 543, 506, 507, 500, 0, 0, 0, 0, 0, 0, 0, 0, 467, 474, 535, 501, 518, 522, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 521, 517, 544, 473, 516, 526, 507, 526, 6, 2, 0, 0, 0, 0, 0, 0, 501, 527, 474, 532, 511, 492, 492, 464, 0, 0, 0, 0, 0, 0, 0, 0, 515, 517, 494, 569, 509, 490, 521, 522, 21, 14, 0, 0, 0, 0, 0, 0, 498, 499, 475, 523, 534, 485, 520, 480, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4088, 4082, 4059, 4138, 3993, 4172, 4050] avg=4096.0000 max=4186 max_rank=0 min=3993 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[483, 470, 503, 495, 519, 512, 486, 483, 1, 0, 0, 0, 0, 0, 0, 0, 483, 519, 491, 522, 552, 472, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 527, 494, 508, 515, 505, 497, 533, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 524, 519, 523, 527, 562, 460, 0, 0, 0, 0, 0, 0, 0, 0, 518, 530, 521, 493, 518, 532, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 506, 524, 531, 492, 555, 514, 538, 523, 0, 0, 0, 0, 0, 0, 0, 0, 506, 515, 522, 483, 525, 500, 498, 492, 15, 11, 0, 0, 0, 0, 0, 0, 483, 502, 537, 519, 504, 523, 541, 515, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3952, 4070, 4094, 4141, 4106, 4183, 4067, 4155] avg=4096.0000 max=4183 max_rank=5 min=3952 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16511 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[526, 532, 511, 557, 530, 510, 495, 528, 485, 548, 526, 510, 496, 500, 516, 510, 514, 505, 497, 511, 504, 481, 506, 491, 481, 523, 562, 540, 523, 488, 529, 511, 496, 519, 504, 526, 556, 468, 549, 502, 516, 528, 492, 532, 530, 492, 499, 496, 506, 547, 536, 499, 536, 495, 504, 521, 483, 524, 463, 464, 511, 529, 508, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[504, 500, 534, 519, 537, 521, 554, 487, 548, 531, 499, 524, 562, 504, 499, 519, 536, 517, 517, 514, 580, 520, 493, 501, 514, 493, 503, 550, 508, 505, 494, 524, 511, 530, 486, 485, 495, 476, 531, 492, 482, 516, 499, 508, 519, 509, 488, 483, 528, 504, 484, 503, 473, 518, 499, 527, 509, 525, 516, 518, 497, 521, 487, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[526, 510, 501, 540, 537, 501, 504, 513, 517, 519, 484, 483, 539, 499, 513, 516, 487, 511, 539, 508, 524, 564, 514, 561, 511, 473, 504, 486, 483, 522, 483, 529, 541, 537, 477, 522, 530, 481, 514, 523, 485, 562, 522, 503, 506, 514, 511, 517, 496, 501, 516, 523, 531, 527, 477, 496, 516, 515, 476, 527, 513, 513, 502, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[490, 517, 546, 524, 513, 490, 524, 500, 553, 497, 500, 520, 497, 548, 541, 574, 525, 556, 502, 520, 501, 491, 534, 497, 492, 479, 489, 494, 514, 542, 513, 515, 514, 526, 496, 535, 510, 535, 510, 494, 520, 512, 518, 480, 501, 484, 527, 519, 494, 538, 499, 547, 487, 529, 456, 487, 503, 522, 513, 513, 518, 488, 492, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[511, 526, 536, 526, 519, 519, 517, 539, 501, 522, 522, 474, 475, 522, 524, 521, 486, 507, 515, 499, 500, 453, 493, 525, 449, 514, 481, 530, 496, 546, 526, 505, 528, 553, 507, 526, 537, 495, 484, 522, 522, 517, 519, 515, 495, 519, 571, 525, 529, 496, 481, 534, 503, 561, 511, 482, 508, 510, 547, 491, 500, 542, 467, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[488, 535, 537, 490, 537, 538, 515, 469, 485, 513, 553, 521, 501, 519, 519, 511, 526, 528, 533, 495, 462, 504, 519, 516, 504, 528, 510, 514, 491, 548, 509, 495, 544, 513, 494, 499, 548, 512, 511, 495, 520, 505, 517, 512, 478, 512, 514, 494, 552, 492, 508, 509, 493, 534, 522, 491, 498, 520, 514, 554, 545, 479, 474, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[532, 518, 532, 516, 506, 557, 520, 485, 526, 527, 513, 499, 511, 522, 515, 546, 538, 484, 531, 485, 495, 505, 495, 511, 521, 515, 530, 487, 500, 499, 503, 542, 566, 496, 491, 534, 506, 497, 496, 489, 505, 542, 536, 489, 518, 500, 509, 534, 511, 483, 470, 506, 504, 513, 491, 505, 513, 527, 482, 525, 514, 512, 557, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[584, 534, 508, 511, 524, 537, 526, 496, 522, 500, 477, 512, 476, 543, 534, 468, 529, 512, 542, 491, 508, 521, 525, 531, 527, 533, 552, 503, 519, 505, 507, 508, 542, 509, 513, 512, 499, 506, 529, 513, 508, 507, 504, 508, 481, 514, 516, 496, 522, 479, 533, 511, 505, 470, 499, 535, 498, 540, 502, 480, 521, 478, 467, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 532, 451, 557, 530, 510, 495, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 548, 526, 510, 496, 500, 516, 490, 0, 0, 0, 0, 0, 0, 0, 0, 514, 505, 495, 511, 504, 481, 506, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 523, 562, 540, 523, 488, 529, 511, 8, 2, 0, 0, 0, 0, 0, 0, 488, 519, 504, 526, 556, 468, 549, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 528, 492, 532, 530, 492, 499, 496, 11, 0, 0, 0, 0, 0, 0, 0, 506, 547, 536, 499, 536, 495, 504, 521, 20, 9, 0, 0, 0, 0, 0, 0, 483, 524, 463, 464, 511, 529, 508, 491, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4071, 4007, 4167, 4112, 4096, 4173, 4013] avg=4096.0000 max=4173 max_rank=6 min=4007 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[526, 510, 442, 540, 537, 501, 504, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 484, 483, 539, 499, 513, 496, 0, 0, 0, 0, 0, 0, 0, 0, 487, 511, 537, 508, 524, 564, 514, 561, 0, 0, 0, 0, 0, 0, 0, 0, 511, 473, 504, 486, 483, 522, 483, 529, 8, 2, 0, 0, 0, 0, 0, 0, 533, 537, 477, 522, 530, 481, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 485, 562, 522, 503, 506, 514, 511, 517, 11, 0, 0, 0, 0, 0, 0, 0, 496, 501, 516, 523, 531, 527, 477, 496, 20, 9, 0, 0, 0, 0, 0, 0, 516, 515, 476, 527, 513, 513, 502, 493, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4050, 4206, 4001, 4117, 4131, 4096, 4094] avg=4096.0000 max=4206 max_rank=2 min=4001 min_rank=3 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[490, 517, 482, 524, 513, 490, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 500, 520, 497, 548, 541, 550, 0, 0, 0, 0, 0, 0, 0, 0, 525, 556, 500, 520, 501, 491, 534, 497, 0, 0, 0, 0, 0, 0, 0, 0, 492, 479, 489, 494, 514, 542, 513, 515, 8, 2, 1, 0, 0, 0, 0, 0, 506, 526, 496, 535, 510, 535, 510, 494, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 518, 480, 501, 484, 527, 519, 13, 0, 0, 0, 0, 0, 0, 0, 494, 538, 499, 547, 487, 529, 456, 487, 22, 10, 0, 0, 0, 0, 0, 0, 503, 522, 513, 513, 518, 488, 492, 503, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4206, 4124, 4049, 4112, 4074, 4069, 4094] avg=4096.0000 max=4206 max_rank=1 min=4040 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[488, 535, 474, 490, 537, 538, 515, 469, 0, 0, 0, 0, 0, 0, 0, 0, 485, 513, 553, 521, 501, 519, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 526, 528, 531, 495, 462, 504, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 510, 514, 491, 548, 509, 495, 8, 2, 0, 0, 0, 0, 0, 0, 536, 513, 494, 499, 548, 512, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 517, 512, 478, 512, 514, 494, 11, 0, 0, 0, 0, 0, 0, 0, 552, 492, 508, 509, 493, 534, 522, 491, 21, 9, 0, 0, 0, 0, 0, 0, 498, 520, 514, 554, 545, 479, 474, 502, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4102, 4081, 4109, 4108, 4063, 4131, 4128] avg=4096.0000 max=4131 max_rank=6 min=4046 min_rank=0 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 500, 471, 519, 537, 521, 554, 487, 0, 0, 0, 0, 0, 0, 0, 0, 548, 531, 499, 524, 562, 504, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 536, 517, 515, 514, 580, 520, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 514, 493, 503, 550, 508, 505, 494, 524, 8, 2, 0, 0, 0, 0, 0, 0, 503, 530, 486, 485, 495, 476, 531, 492, 0, 0, 0, 0, 0, 0, 0, 0, 482, 516, 499, 508, 519, 509, 488, 483, 11, 0, 0, 0, 0, 0, 0, 0, 528, 504, 484, 503, 473, 518, 499, 527, 21, 9, 0, 0, 0, 0, 0, 0, 509, 525, 516, 518, 497, 521, 487, 538, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4166, 4176, 4101, 3998, 4015, 4066, 4153] avg=4096.0000 max=4176 max_rank=2 min=3998 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16232 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[532, 518, 470, 516, 506, 557, 520, 485, 0, 0, 0, 0, 0, 0, 0, 0, 526, 527, 513, 499, 511, 522, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 538, 484, 529, 485, 495, 505, 495, 511, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 530, 487, 500, 499, 503, 542, 9, 2, 1, 0, 0, 0, 0, 0, 557, 496, 491, 534, 506, 497, 496, 489, 0, 0, 0, 0, 0, 0, 0, 0, 505, 542, 536, 489, 518, 500, 509, 534, 12, 0, 0, 0, 0, 0, 0, 0, 511, 483, 470, 506, 504, 513, 491, 505, 21, 10, 0, 0, 0, 0, 0, 0, 513, 527, 482, 525, 514, 512, 557, 481, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4136, 4042, 4109, 4066, 4145, 4014, 4152] avg=4096.0000 max=4152 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 526, 465, 526, 519, 519, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 501, 522, 522, 474, 475, 522, 524, 483, 0, 0, 0, 0, 0, 0, 0, 0, 486, 507, 512, 499, 500, 453, 493, 525, 0, 0, 0, 0, 0, 0, 0, 0, 449, 514, 481, 530, 496, 546, 526, 505, 12, 3, 6, 0, 0, 0, 0, 0, 516, 553, 507, 526, 537, 495, 484, 522, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 519, 515, 495, 519, 571, 525, 17, 0, 0, 0, 0, 0, 0, 0, 529, 496, 481, 534, 503, 561, 511, 482, 25, 15, 0, 0, 0, 0, 0, 0, 508, 510, 547, 491, 500, 542, 467, 492, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4023, 3975, 4068, 4140, 4200, 4137, 4103] avg=4096.0000 max=4200 max_rank=5 min=3975 min_rank=2 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[584, 534, 449, 511, 524, 537, 526, 496, 0, 0, 0, 0, 0, 0, 0, 0, 522, 500, 477, 512, 476, 543, 534, 450, 0, 0, 0, 0, 0, 0, 0, 0, 529, 512, 540, 491, 508, 521, 525, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 533, 552, 503, 519, 505, 507, 508, 8, 2, 0, 0, 0, 0, 0, 0, 534, 509, 513, 512, 499, 506, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 504, 508, 481, 514, 516, 496, 10, 0, 0, 0, 0, 0, 0, 0, 522, 479, 533, 511, 505, 470, 499, 535, 20, 8, 0, 0, 0, 0, 0, 0, 498, 540, 502, 480, 521, 478, 467, 506, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4014, 4157, 4164, 4115, 4044, 4082, 4031] avg=4096.0000 max=4164 max_rank=3 min=4014 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16272 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 3968, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 3712, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 3968, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[571, 528, 501, 545, 507, 489, 501, 492, 505, 551, 524, 527, 533, 527, 527, 488, 500, 506, 521, 525, 534, 485, 502, 494, 506, 514, 500, 489, 511, 510, 515, 505, 511, 517, 529, 519, 516, 526, 489, 510, 500, 513, 504, 516, 503, 507, 523, 511, 518, 500, 512, 519, 461, 505, 529, 493, 533, 524, 503, 494, 510, 494, 517, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[513, 507, 481, 557, 515, 537, 535, 517, 517, 474, 510, 468, 495, 535, 522, 517, 489, 526, 548, 550, 539, 546, 511, 508, 523, 511, 477, 515, 480, 499, 536, 511, 506, 518, 509, 502, 467, 524, 499, 516, 480, 504, 546, 495, 514, 534, 530, 459, 535, 515, 510, 502, 528, 507, 512, 467, 500, 511, 463, 534, 506, 540, 561, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[506, 539, 508, 519, 538, 508, 498, 507, 499, 494, 501, 503, 544, 519, 508, 512, 532, 572, 515, 512, 497, 539, 505, 531, 485, 450, 545, 538, 539, 507, 522, 514, 501, 508, 499, 507, 524, 469, 478, 517, 527, 510, 506, 539, 491, 497, 536, 515, 507, 511, 515, 526, 495, 517, 500, 506, 508, 533, 519, 518, 499, 502, 517, 465] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[487, 511, 517, 518, 496, 515, 541, 509, 465, 521, 495, 484, 488, 523, 485, 540, 518, 507, 528, 533, 490, 527, 520, 501, 526, 535, 492, 536, 515, 565, 495, 500, 488, 546, 492, 541, 518, 493, 507, 513, 509, 528, 526, 496, 504, 489, 533, 497, 527, 508, 506, 507, 529, 493, 493, 525, 492, 485, 520, 512, 494, 514, 576, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[473, 517, 503, 531, 500, 489, 506, 503, 510, 542, 548, 547, 503, 540, 523, 525, 498, 475, 510, 525, 534, 494, 518, 525, 490, 508, 477, 523, 512, 519, 515, 513, 519, 533, 536, 522, 497, 524, 460, 536, 507, 505, 492, 528, 509, 528, 447, 520, 547, 504, 494, 508, 487, 516, 563, 478, 554, 541, 477, 496, 519, 531, 503, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[512, 518, 507, 505, 512, 515, 516, 539, 498, 530, 508, 509, 516, 506, 546, 518, 544, 492, 502, 489, 479, 526, 510, 529, 544, 565, 522, 541, 500, 490, 491, 517, 521, 533, 505, 508, 483, 493, 503, 516, 506, 486, 498, 484, 490, 561, 523, 523, 467, 505, 507, 536, 516, 501, 502, 467, 554, 499, 533, 541, 504, 519, 498, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[518, 538, 488, 521, 530, 501, 495, 491, 545, 511, 503, 502, 488, 491, 526, 536, 502, 534, 549, 489, 530, 510, 500, 535, 514, 511, 515, 472, 529, 485, 522, 508, 509, 516, 507, 497, 525, 506, 513, 521, 509, 490, 506, 542, 486, 525, 511, 490, 484, 492, 487, 514, 522, 508, 549, 520, 472, 506, 527, 524, 521, 522, 536, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[560, 471, 514, 520, 552, 512, 531, 507, 501, 544, 521, 535, 526, 539, 460, 494, 497, 548, 519, 511, 497, 533, 512, 491, 496, 542, 503, 500, 528, 506, 473, 525, 485, 547, 560, 519, 536, 476, 503, 507, 533, 523, 508, 501, 465, 492, 511, 524, 495, 542, 467, 506, 513, 508, 502, 525, 527, 509, 504, 463, 533, 513, 504, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[571, 528, 501, 527, 507, 489, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 505, 551, 524, 527, 533, 513, 527, 488, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 490, 525, 534, 485, 502, 494, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 500, 489, 511, 510, 515, 505, 5, 0, 0, 0, 0, 0, 0, 0, 511, 517, 529, 519, 516, 526, 489, 510, 6, 4, 0, 0, 0, 0, 0, 0, 500, 513, 504, 516, 503, 507, 523, 511, 17, 8, 0, 0, 0, 0, 0, 0, 518, 500, 512, 519, 461, 505, 529, 493, 31, 1, 0, 0, 0, 0, 0, 0, 533, 524, 503, 494, 510, 494, 508, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4168, 4036, 4055, 4127, 4102, 4069, 4095] avg=4096.0000 max=4168 max_rank=1 min=4036 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 507, 481, 539, 515, 537, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 517, 474, 510, 468, 495, 520, 522, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 526, 516, 550, 539, 546, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 477, 515, 480, 499, 536, 511, 6, 0, 0, 0, 0, 0, 0, 0, 506, 518, 509, 502, 467, 524, 499, 516, 6, 5, 0, 0, 0, 0, 0, 0, 480, 504, 546, 495, 514, 534, 530, 459, 17, 9, 0, 0, 0, 0, 0, 0, 535, 515, 510, 502, 528, 507, 512, 467, 32, 1, 0, 0, 0, 0, 0, 0, 500, 511, 463, 534, 506, 540, 550, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4023, 4185, 4058, 4052, 4088, 4109, 4109] avg=4096.0000 max=4185 max_rank=2 min=4023 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[506, 539, 508, 503, 538, 508, 498, 507, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 501, 503, 544, 505, 508, 512, 0, 0, 0, 0, 0, 0, 0, 0, 532, 572, 485, 512, 497, 539, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 485, 450, 545, 538, 539, 507, 522, 514, 5, 0, 0, 0, 0, 0, 0, 0, 501, 508, 499, 507, 524, 469, 478, 517, 6, 4, 0, 0, 0, 0, 0, 0, 527, 510, 506, 539, 491, 497, 536, 515, 16, 8, 0, 0, 0, 0, 0, 0, 507, 511, 515, 526, 495, 517, 500, 506, 30, 0, 0, 0, 0, 0, 0, 0, 508, 533, 519, 518, 499, 502, 508, 465, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4066, 4173, 4105, 4013, 4145, 4107, 4052] avg=4096.0000 max=4173 max_rank=2 min=4013 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[518, 538, 488, 505, 530, 501, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 545, 511, 503, 502, 488, 477, 526, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 534, 517, 489, 530, 510, 500, 535, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 515, 472, 529, 485, 522, 508, 5, 0, 0, 0, 0, 0, 0, 0, 509, 516, 507, 497, 525, 506, 513, 521, 6, 5, 0, 0, 0, 0, 0, 0, 509, 490, 506, 542, 486, 525, 511, 490, 16, 8, 0, 0, 0, 0, 0, 0, 484, 492, 487, 514, 522, 508, 549, 520, 32, 0, 0, 0, 0, 0, 0, 0, 472, 506, 527, 524, 521, 522, 526, 542, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4088, 4117, 4061, 4105, 4083, 4108, 4140] avg=4096.0000 max=4140 max_rank=7 min=4061 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[487, 511, 517, 502, 496, 515, 541, 509, 0, 0, 0, 0, 0, 0, 0, 0, 465, 521, 495, 484, 488, 509, 485, 540, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 497, 533, 490, 527, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 526, 535, 492, 536, 515, 565, 495, 500, 6, 0, 0, 0, 0, 0, 0, 0, 488, 546, 492, 541, 518, 493, 507, 513, 6, 5, 0, 0, 0, 0, 0, 0, 509, 528, 526, 496, 504, 489, 533, 497, 16, 8, 0, 0, 0, 0, 0, 0, 527, 508, 506, 507, 529, 493, 493, 525, 31, 0, 0, 0, 0, 0, 0, 0, 492, 485, 520, 512, 494, 514, 565, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3987, 4093, 4170, 4109, 4106, 4119, 4106] avg=4096.0000 max=4170 max_rank=3 min=3987 min_rank=1 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[560, 471, 514, 504, 552, 512, 531, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 544, 521, 535, 526, 524, 460, 494, 0, 0, 0, 0, 0, 0, 0, 0, 497, 548, 489, 511, 497, 533, 512, 491, 0, 0, 0, 0, 0, 0, 0, 0, 496, 542, 503, 500, 528, 506, 473, 525, 5, 0, 0, 0, 0, 0, 0, 0, 485, 547, 560, 519, 536, 476, 503, 507, 6, 4, 0, 0, 0, 0, 0, 0, 533, 523, 508, 501, 465, 492, 511, 524, 16, 9, 0, 0, 0, 0, 0, 0, 495, 542, 467, 506, 513, 508, 502, 525, 30, 0, 0, 0, 0, 0, 0, 0, 527, 509, 504, 463, 533, 513, 495, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4105, 4078, 4078, 4143, 4082, 4088, 4043] avg=4096.0000 max=4151 max_rank=0 min=4043 min_rank=7 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16356 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[473, 517, 503, 514, 500, 489, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 510, 542, 548, 547, 503, 525, 523, 525, 0, 0, 0, 0, 0, 0, 0, 0, 498, 475, 480, 525, 534, 494, 518, 525, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 477, 523, 512, 519, 515, 513, 5, 0, 0, 0, 0, 0, 0, 0, 519, 533, 536, 522, 497, 524, 460, 536, 6, 4, 0, 0, 0, 0, 0, 0, 507, 505, 492, 528, 509, 528, 447, 520, 16, 9, 0, 0, 0, 0, 0, 0, 547, 504, 494, 508, 487, 516, 563, 478, 30, 1, 0, 0, 0, 0, 0, 0, 554, 541, 477, 496, 519, 531, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4005, 4223, 4049, 4062, 4137, 4061, 4128, 4103] avg=4096.0000 max=4223 max_rank=1 min=4005 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 518, 507, 482, 512, 515, 516, 539, 0, 0, 0, 0, 0, 0, 0, 0, 498, 530, 508, 509, 516, 483, 546, 518, 0, 0, 0, 0, 0, 0, 0, 0, 544, 492, 468, 489, 479, 526, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 544, 565, 522, 541, 500, 490, 491, 517, 9, 0, 0, 0, 0, 0, 0, 0, 521, 533, 505, 508, 483, 493, 503, 516, 11, 9, 0, 0, 0, 0, 0, 0, 506, 486, 498, 484, 490, 561, 523, 523, 18, 12, 0, 0, 0, 0, 0, 0, 467, 505, 507, 536, 516, 501, 502, 467, 34, 5, 0, 0, 0, 0, 0, 0, 554, 499, 533, 541, 504, 519, 480, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4108, 4037, 4179, 4082, 4101, 4040, 4120] avg=4096.0000 max=4179 max_rank=3 min=4037 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 3968, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[529, 476, 517, 500, 494, 551, 536, 550, 501, 486, 527, 513, 517, 541, 513, 486, 442, 499, 512, 523, 548, 534, 537, 540, 511, 527, 515, 530, 523, 502, 520, 516, 560, 508, 517, 536, 473, 513, 518, 507, 502, 467, 531, 477, 522, 484, 468, 486, 492, 522, 485, 502, 542, 497, 511, 508, 500, 530, 495, 528, 548, 498, 521, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[483, 462, 516, 517, 525, 490, 539, 498, 505, 518, 537, 504, 551, 518, 524, 538, 508, 532, 540, 571, 507, 511, 501, 506, 501, 492, 486, 485, 489, 511, 514, 503, 510, 500, 507, 518, 510, 498, 546, 532, 544, 503, 509, 514, 538, 498, 508, 521, 469, 502, 500, 517, 509, 523, 513, 566, 507, 505, 477, 487, 514, 565, 480, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[487, 498, 467, 505, 499, 508, 525, 537, 524, 536, 511, 507, 519, 509, 485, 510, 488, 531, 535, 512, 550, 543, 520, 459, 517, 511, 495, 511, 529, 549, 514, 495, 500, 540, 468, 516, 477, 547, 518, 522, 484, 529, 542, 519, 558, 474, 510, 512, 492, 507, 516, 516, 547, 530, 484, 522, 520, 481, 521, 493, 535, 494, 491, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[486, 507, 492, 512, 516, 490, 522, 549, 544, 529, 530, 509, 533, 497, 530, 506, 531, 505, 492, 500, 523, 518, 503, 544, 501, 566, 483, 508, 503, 533, 523, 503, 517, 503, 528, 516, 501, 534, 492, 489, 528, 495, 536, 495, 532, 510, 452, 544, 526, 518, 475, 506, 502, 478, 527, 498, 512, 525, 505, 500, 511, 484, 520, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[487, 505, 521, 501, 535, 530, 536, 526, 495, 503, 512, 526, 502, 518, 503, 515, 453, 503, 531, 487, 501, 531, 500, 516, 520, 533, 488, 472, 520, 524, 522, 464, 504, 517, 525, 521, 538, 491, 509, 468, 526, 483, 527, 532, 521, 509, 502, 524, 508, 504, 520, 551, 505, 552, 518, 510, 558, 493, 520, 488, 532, 511, 511, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[542, 527, 487, 532, 504, 512, 496, 520, 558, 520, 503, 510, 509, 483, 510, 531, 509, 504, 494, 514, 484, 512, 533, 521, 517, 501, 538, 514, 504, 547, 497, 514, 496, 501, 490, 522, 503, 520, 515, 525, 533, 523, 534, 498, 519, 517, 519, 501, 503, 521, 482, 515, 472, 517, 531, 488, 515, 518, 505, 536, 504, 480, 495, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[530, 551, 516, 524, 526, 526, 513, 476, 492, 514, 539, 556, 521, 548, 533, 514, 499, 499, 503, 537, 524, 505, 506, 486, 519, 545, 529, 483, 473, 523, 506, 496, 521, 502, 498, 516, 547, 500, 454, 544, 493, 478, 510, 487, 526, 512, 515, 520, 516, 511, 482, 498, 486, 523, 551, 497, 536, 509, 541, 506, 506, 506, 472, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[507, 483, 509, 494, 451, 525, 453, 484, 547, 511, 515, 524, 507, 516, 534, 512, 498, 529, 525, 534, 484, 544, 504, 501, 494, 553, 498, 555, 493, 507, 504, 502, 503, 501, 530, 551, 505, 498, 480, 486, 497, 516, 524, 517, 528, 516, 510, 489, 586, 514, 558, 519, 497, 464, 474, 523, 518, 544, 551, 513, 546, 546, 483, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[529, 476, 517, 500, 494, 551, 536, 550, 19, 0, 0, 0, 0, 0, 0, 0, 501, 486, 483, 513, 517, 541, 513, 486, 0, 0, 0, 0, 0, 0, 0, 0, 442, 499, 512, 523, 548, 527, 537, 540, 0, 0, 0, 0, 0, 0, 0, 0, 511, 527, 515, 530, 523, 502, 520, 516, 4, 1, 0, 0, 0, 0, 0, 0, 560, 508, 517, 536, 473, 513, 518, 507, 8, 0, 0, 0, 0, 0, 0, 0, 502, 467, 531, 477, 522, 484, 468, 486, 8, 0, 0, 0, 0, 0, 0, 0, 492, 522, 485, 502, 542, 497, 511, 508, 8, 0, 0, 0, 0, 0, 0, 0, 500, 530, 495, 528, 548, 498, 521, 504, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4040, 4128, 4149, 4140, 3945, 4067, 4127] avg=4096.0000 max=4172 max_rank=0 min=3945 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 462, 516, 517, 525, 490, 539, 498, 20, 0, 0, 0, 0, 0, 0, 0, 505, 518, 490, 504, 551, 518, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 540, 571, 507, 504, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 492, 486, 485, 489, 511, 514, 503, 4, 1, 0, 0, 0, 0, 0, 0, 510, 500, 507, 518, 510, 498, 546, 532, 8, 0, 0, 0, 0, 0, 0, 0, 544, 503, 509, 514, 538, 498, 508, 521, 9, 0, 0, 0, 0, 0, 0, 0, 469, 502, 500, 517, 509, 523, 513, 566, 9, 0, 0, 0, 0, 0, 0, 0, 507, 505, 477, 487, 514, 565, 480, 496, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4148, 4169, 3986, 4129, 4144, 4108, 4034] avg=4096.0000 max=4169 max_rank=2 min=3986 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 505, 521, 501, 535, 530, 536, 526, 19, 0, 0, 0, 0, 0, 0, 0, 495, 503, 468, 526, 502, 518, 503, 515, 0, 0, 0, 0, 0, 0, 0, 0, 453, 503, 531, 487, 501, 524, 500, 516, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 488, 472, 520, 524, 522, 464, 4, 1, 0, 0, 0, 0, 0, 0, 504, 517, 525, 521, 538, 491, 509, 468, 8, 0, 0, 0, 0, 0, 0, 0, 526, 483, 527, 532, 521, 509, 502, 524, 8, 0, 0, 0, 0, 0, 0, 0, 508, 504, 520, 551, 505, 552, 518, 510, 8, 0, 0, 0, 0, 0, 0, 0, 558, 493, 520, 488, 532, 511, 511, 510, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4160, 4030, 4015, 4048, 4081, 4132, 4176, 4126] avg=4096.0000 max=4176 max_rank=6 min=4015 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 498, 467, 505, 499, 508, 525, 537, 19, 0, 0, 0, 0, 0, 0, 0, 524, 536, 467, 507, 519, 509, 485, 510, 0, 0, 0, 0, 0, 0, 0, 0, 488, 531, 535, 512, 550, 536, 520, 459, 0, 0, 0, 0, 0, 0, 0, 0, 517, 511, 495, 511, 529, 549, 514, 495, 4, 1, 0, 0, 0, 0, 0, 0, 500, 540, 468, 516, 477, 547, 518, 522, 8, 0, 0, 0, 0, 0, 0, 0, 484, 529, 542, 519, 558, 474, 510, 512, 8, 0, 0, 0, 0, 0, 0, 0, 492, 507, 516, 516, 547, 530, 484, 522, 8, 0, 0, 0, 0, 0, 0, 0, 520, 481, 521, 493, 535, 494, 491, 517, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4057, 4131, 4126, 4096, 4136, 4122, 4055] avg=4096.0000 max=4136 max_rank=5 min=4045 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[486, 507, 492, 512, 516, 490, 522, 549, 19, 0, 0, 0, 0, 0, 0, 0, 544, 529, 486, 509, 533, 497, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 531, 505, 492, 500, 523, 511, 503, 544, 0, 0, 0, 0, 0, 0, 0, 0, 501, 566, 483, 508, 503, 533, 523, 503, 4, 1, 0, 0, 0, 0, 0, 0, 517, 503, 528, 516, 501, 534, 492, 489, 8, 0, 0, 0, 0, 0, 0, 0, 528, 495, 536, 495, 532, 510, 452, 544, 8, 0, 0, 0, 0, 0, 0, 0, 526, 518, 475, 506, 502, 478, 527, 498, 8, 0, 0, 0, 0, 0, 0, 0, 512, 525, 505, 500, 511, 484, 520, 521, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4134, 4109, 4125, 4088, 4100, 4038, 4081] avg=4096.0000 max=4134 max_rank=1 min=4038 min_rank=6 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[507, 483, 509, 494, 451, 525, 453, 484, 19, 0, 0, 0, 0, 0, 0, 0, 547, 511, 471, 524, 507, 516, 534, 512, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 525, 534, 484, 537, 504, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 553, 498, 555, 493, 507, 504, 502, 4, 1, 0, 0, 0, 0, 0, 0, 503, 501, 530, 551, 505, 498, 480, 486, 8, 0, 0, 0, 0, 0, 0, 0, 497, 516, 524, 517, 528, 516, 510, 489, 8, 0, 0, 0, 0, 0, 0, 0, 586, 514, 558, 519, 497, 464, 474, 523, 8, 0, 0, 0, 0, 0, 0, 0, 518, 544, 551, 513, 546, 546, 483, 484, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3925, 4122, 4112, 4111, 4062, 4105, 4143, 4188] avg=4096.0000 max=4188 max_rank=7 min=3925 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16498 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 551, 516, 524, 526, 526, 513, 476, 20, 0, 0, 0, 0, 0, 0, 0, 492, 514, 492, 556, 521, 548, 533, 514, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 503, 537, 524, 498, 506, 486, 0, 0, 0, 0, 0, 0, 0, 0, 519, 545, 529, 483, 473, 523, 506, 496, 4, 1, 0, 0, 0, 0, 0, 0, 521, 502, 498, 516, 547, 500, 454, 544, 8, 0, 0, 0, 0, 0, 0, 0, 493, 478, 510, 487, 526, 512, 515, 520, 9, 0, 0, 0, 0, 0, 0, 0, 516, 511, 482, 498, 486, 523, 551, 497, 9, 0, 0, 0, 0, 0, 0, 0, 536, 509, 541, 506, 506, 506, 472, 493, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4170, 4052, 4079, 4090, 4050, 4073, 4072] avg=4096.0000 max=4182 max_rank=0 min=4050 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[542, 527, 487, 532, 504, 512, 496, 520, 21, 0, 0, 0, 0, 0, 0, 0, 558, 520, 446, 510, 509, 483, 510, 531, 0, 0, 0, 0, 0, 0, 0, 0, 509, 504, 494, 514, 484, 493, 533, 521, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 538, 514, 504, 547, 497, 514, 10, 2, 0, 0, 0, 0, 0, 0, 496, 501, 490, 522, 503, 520, 515, 525, 10, 0, 0, 0, 0, 0, 0, 0, 533, 523, 534, 498, 519, 517, 519, 501, 12, 0, 0, 0, 0, 0, 0, 0, 503, 521, 482, 515, 472, 517, 531, 488, 12, 0, 0, 0, 0, 0, 0, 0, 515, 518, 505, 536, 504, 480, 495, 523, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4067, 4052, 4144, 4082, 4156, 4041, 4085] avg=4096.0000 max=4156 max_rank=5 min=4041 min_rank=6 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4352, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[515, 551, 501, 505, 538, 534, 539, 505, 500, 489, 508, 487, 480, 508, 509, 509, 531, 505, 475, 487, 502, 500, 495, 509, 589, 524, 501, 514, 529, 555, 523, 542, 510, 527, 517, 543, 485, 505, 547, 490, 525, 486, 499, 530, 519, 515, 519, 504, 537, 510, 507, 525, 503, 515, 513, 494, 479, 481, 521, 505, 502, 494, 484, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[522, 531, 502, 509, 491, 521, 468, 513, 515, 536, 547, 517, 520, 492, 494, 511, 498, 526, 502, 504, 529, 468, 525, 492, 504, 513, 483, 527, 519, 489, 501, 543, 488, 480, 548, 545, 508, 551, 536, 513, 505, 534, 537, 547, 484, 551, 515, 485, 502, 530, 533, 478, 535, 478, 555, 506, 510, 513, 522, 466, 486, 526, 492, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[521, 528, 495, 522, 502, 535, 518, 495, 509, 529, 518, 526, 501, 476, 537, 491, 531, 526, 518, 472, 490, 521, 490, 534, 487, 595, 533, 513, 514, 522, 527, 502, 518, 495, 521, 548, 518, 520, 530, 504, 523, 481, 529, 516, 486, 485, 507, 491, 522, 490, 554, 481, 533, 466, 522, 508, 506, 481, 522, 505, 496, 515, 508, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[555, 505, 524, 524, 558, 494, 509, 522, 494, 501, 499, 498, 485, 483, 533, 509, 539, 526, 526, 552, 502, 475, 524, 559, 525, 497, 502, 493, 546, 514, 537, 489, 493, 496, 506, 490, 567, 522, 520, 538, 538, 517, 496, 487, 518, 506, 555, 502, 500, 513, 479, 537, 482, 526, 517, 516, 521, 497, 546, 492, 493, 458, 480, 461] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[495, 515, 542, 550, 478, 527, 505, 466, 492, 494, 523, 480, 527, 550, 495, 496, 510, 533, 513, 488, 594, 506, 526, 515, 537, 501, 519, 500, 516, 499, 545, 489, 532, 538, 547, 554, 520, 471, 500, 526, 533, 514, 499, 486, 530, 532, 516, 491, 483, 484, 537, 481, 494, 544, 511, 490, 514, 486, 496, 502, 495, 516, 513, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[509, 513, 575, 520, 537, 527, 516, 532, 532, 460, 513, 502, 529, 480, 488, 545, 519, 558, 528, 517, 491, 518, 483, 541, 505, 482, 493, 514, 481, 510, 503, 533, 503, 504, 552, 497, 514, 501, 499, 487, 484, 511, 479, 528, 502, 507, 519, 534, 554, 570, 492, 551, 535, 526, 533, 510, 487, 484, 523, 483, 493, 460, 514, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[513, 508, 537, 515, 533, 513, 536, 546, 526, 520, 501, 502, 509, 509, 520, 514, 521, 520, 508, 547, 511, 509, 479, 537, 509, 527, 514, 529, 500, 501, 529, 502, 504, 507, 491, 529, 534, 486, 507, 496, 508, 539, 514, 475, 534, 504, 513, 511, 511, 490, 526, 504, 506, 526, 522, 473, 516, 494, 555, 453, 519, 492, 495, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[525, 543, 516, 517, 514, 505, 532, 529, 511, 538, 498, 492, 512, 515, 504, 500, 517, 519, 495, 504, 557, 511, 491, 526, 512, 530, 495, 511, 538, 522, 526, 529, 521, 516, 467, 558, 564, 517, 501, 533, 515, 525, 502, 488, 495, 483, 489, 484, 500, 518, 489, 518, 510, 518, 504, 553, 524, 491, 509, 464, 493, 482, 500, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 469, 495, 522, 502, 535, 518, 495, 0, 0, 0, 0, 0, 0, 0, 0, 509, 529, 518, 526, 501, 476, 537, 491, 24, 11, 0, 0, 0, 0, 0, 0, 531, 526, 518, 472, 490, 521, 490, 515, 0, 0, 0, 0, 0, 0, 0, 0, 487, 595, 533, 513, 514, 522, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 495, 521, 512, 518, 520, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 523, 481, 529, 516, 486, 485, 507, 491, 8, 7, 0, 0, 0, 0, 0, 0, 522, 490, 554, 481, 533, 466, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 506, 481, 522, 505, 496, 515, 508, 509, 59, 36, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4122, 4063, 4158, 4118, 4033, 4069, 4148] avg=4096.0000 max=4158 max_rank=3 min=4033 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[555, 449, 524, 524, 558, 494, 509, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 499, 498, 485, 483, 533, 509, 24, 11, 0, 0, 0, 0, 0, 0, 539, 526, 526, 552, 502, 475, 524, 539, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 502, 493, 546, 514, 502, 489, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 506, 458, 567, 522, 520, 538, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 496, 487, 518, 506, 555, 502, 9, 7, 0, 0, 0, 0, 0, 0, 500, 513, 479, 537, 482, 526, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 497, 546, 492, 493, 458, 480, 461, 56, 32, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4037, 4183, 4068, 4100, 4135, 4063, 4047] avg=4096.0000 max=4183 max_rank=2 min=4037 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 489, 501, 505, 538, 534, 539, 505, 0, 0, 0, 0, 0, 0, 0, 0, 500, 489, 508, 487, 480, 508, 509, 509, 24, 10, 0, 0, 0, 0, 0, 0, 531, 505, 475, 487, 502, 500, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 589, 524, 501, 514, 529, 555, 488, 542, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 517, 507, 485, 505, 547, 490, 0, 0, 0, 0, 0, 0, 0, 0, 525, 486, 499, 530, 519, 515, 519, 504, 8, 7, 0, 0, 0, 0, 0, 0, 537, 510, 507, 525, 503, 515, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 479, 481, 521, 505, 502, 494, 484, 518, 62, 36, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4024, 3986, 4242, 4088, 4112, 4097, 4093] avg=4096.0000 max=4242 max_rank=3 min=3986 min_rank=2 imbalance_ratio=1.035645 max_load_reduction=28526 max_load_reduction_pct=0.870544 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 456, 575, 520, 537, 527, 516, 532, 0, 0, 0, 0, 0, 0, 0, 0, 532, 460, 513, 502, 529, 480, 488, 545, 23, 11, 0, 0, 0, 0, 0, 0, 519, 558, 528, 517, 491, 518, 483, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 482, 493, 514, 481, 510, 469, 533, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 552, 464, 514, 501, 499, 487, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 479, 528, 502, 507, 519, 534, 8, 7, 0, 0, 0, 0, 0, 0, 554, 570, 492, 551, 535, 526, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 487, 484, 523, 483, 493, 460, 514, 478, 57, 33, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4083, 4136, 3987, 4024, 4079, 4264, 4023] avg=4096.0000 max=4264 max_rank=6 min=3987 min_rank=3 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 472, 502, 509, 491, 521, 468, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 536, 547, 517, 520, 492, 494, 511, 23, 10, 0, 0, 0, 0, 0, 0, 498, 526, 502, 504, 529, 468, 525, 474, 0, 0, 0, 0, 0, 0, 0, 0, 504, 513, 483, 527, 519, 489, 468, 543, 0, 0, 0, 0, 0, 0, 0, 0, 488, 480, 548, 509, 508, 551, 536, 513, 0, 0, 0, 0, 0, 0, 0, 0, 505, 534, 537, 547, 484, 551, 515, 485, 8, 8, 0, 0, 0, 0, 0, 0, 502, 530, 533, 478, 535, 478, 547, 506, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 522, 466, 486, 526, 492, 497, 59, 36, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4165, 4026, 4046, 4133, 4174, 4109, 4117] avg=4096.0000 max=4174 max_rank=5 min=3998 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16533 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 482, 516, 517, 514, 505, 532, 529, 0, 0, 0, 0, 0, 0, 0, 0, 511, 538, 498, 492, 512, 515, 504, 500, 24, 10, 0, 0, 0, 0, 0, 0, 517, 519, 495, 504, 557, 511, 491, 508, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 495, 511, 538, 522, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0, 521, 516, 467, 521, 564, 517, 501, 533, 0, 0, 0, 0, 0, 0, 0, 0, 515, 525, 502, 488, 495, 483, 489, 484, 8, 7, 0, 0, 0, 0, 0, 0, 500, 518, 489, 518, 510, 518, 497, 553, 0, 0, 0, 0, 0, 0, 0, 0, 524, 491, 509, 464, 493, 482, 500, 503, 61, 37, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4104, 4102, 4128, 4140, 3996, 4103, 4075] avg=4096.0000 max=4140 max_rank=4 min=3996 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[495, 453, 542, 550, 478, 527, 505, 466, 0, 0, 0, 0, 0, 0, 0, 0, 492, 494, 523, 480, 527, 550, 495, 496, 27, 13, 0, 0, 0, 0, 0, 0, 510, 533, 513, 488, 594, 506, 526, 489, 0, 0, 0, 0, 0, 0, 0, 0, 537, 501, 519, 500, 516, 499, 502, 489, 0, 0, 0, 0, 0, 0, 0, 0, 532, 538, 547, 512, 520, 471, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 533, 514, 499, 486, 530, 532, 516, 491, 13, 12, 0, 0, 0, 0, 0, 0, 483, 484, 537, 481, 494, 544, 499, 490, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 496, 502, 495, 516, 513, 507, 62, 42, 16, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4016, 4097, 4159, 4063, 4146, 4126, 4012, 4149] avg=4096.0000 max=4159 max_rank=2 min=4012 min_rank=6 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 451, 537, 515, 533, 513, 536, 546, 0, 0, 0, 0, 0, 0, 0, 0, 526, 520, 501, 502, 509, 509, 520, 514, 24, 11, 0, 0, 0, 0, 0, 0, 521, 520, 508, 547, 511, 509, 479, 518, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 514, 529, 500, 501, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 491, 494, 534, 486, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 508, 539, 514, 475, 534, 504, 513, 511, 8, 7, 0, 0, 0, 0, 0, 0, 511, 490, 526, 504, 506, 526, 515, 473, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 555, 453, 519, 492, 495, 489, 57, 35, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4136, 4113, 4076, 4019, 4113, 4051, 4116] avg=4096.0000 max=4144 max_rank=0 min=4019 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 3968, 4096, 3968, 4096, 3968, 512, 384, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[530, 509, 506, 573, 494, 521, 522, 519, 527, 480, 518, 470, 507, 520, 489, 522, 551, 483, 533, 567, 485, 527, 493, 520, 516, 486, 496, 525, 518, 500, 516, 518, 493, 511, 498, 495, 546, 488, 490, 476, 533, 556, 505, 525, 553, 501, 497, 510, 482, 483, 523, 547, 530, 509, 504, 518, 516, 504, 462, 539, 520, 496, 502, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[496, 508, 499, 513, 519, 540, 501, 553, 503, 521, 548, 557, 523, 481, 534, 471, 519, 483, 496, 523, 553, 511, 526, 498, 491, 509, 522, 540, 522, 515, 514, 495, 521, 476, 496, 494, 491, 497, 517, 498, 528, 510, 521, 528, 489, 514, 462, 532, 507, 518, 514, 531, 520, 493, 512, 552, 500, 537, 494, 472, 536, 490, 519, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[459, 506, 507, 537, 514, 516, 479, 513, 524, 531, 521, 510, 539, 513, 548, 526, 538, 530, 473, 504, 503, 527, 514, 506, 492, 522, 541, 521, 513, 499, 518, 537, 514, 521, 483, 522, 528, 502, 468, 518, 537, 510, 546, 471, 532, 522, 537, 528, 521, 533, 472, 535, 522, 493, 519, 535, 475, 492, 457, 513, 508, 498, 488, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[518, 538, 533, 480, 549, 539, 516, 499, 509, 530, 492, 501, 502, 474, 496, 507, 531, 513, 482, 483, 515, 519, 505, 528, 522, 494, 525, 499, 554, 517, 500, 531, 500, 506, 493, 492, 556, 524, 525, 483, 514, 530, 499, 523, 505, 526, 550, 520, 532, 491, 501, 524, 497, 494, 526, 525, 534, 509, 529, 498, 458, 475, 511, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[491, 534, 506, 530, 524, 519, 523, 515, 462, 563, 517, 511, 492, 515, 531, 538, 508, 507, 504, 586, 545, 516, 517, 473, 486, 508, 487, 520, 544, 511, 484, 525, 512, 539, 521, 484, 467, 528, 494, 527, 479, 528, 535, 514, 525, 505, 486, 499, 495, 489, 519, 521, 507, 499, 554, 514, 535, 523, 505, 514, 493, 470, 516, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[495, 534, 519, 492, 499, 515, 527, 514, 511, 508, 490, 495, 486, 552, 483, 474, 522, 530, 523, 544, 513, 489, 519, 528, 552, 547, 496, 497, 504, 531, 477, 514, 501, 491, 493, 505, 540, 483, 496, 511, 546, 526, 450, 489, 510, 542, 507, 514, 481, 564, 502, 511, 531, 543, 536, 515, 535, 499, 508, 521, 517, 520, 500, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[544, 545, 541, 531, 532, 521, 542, 473, 489, 529, 520, 552, 519, 498, 475, 545, 543, 511, 496, 557, 507, 522, 488, 495, 511, 485, 491, 520, 517, 531, 537, 518, 535, 511, 518, 527, 510, 514, 505, 534, 510, 488, 518, 523, 495, 525, 529, 486, 501, 450, 521, 495, 533, 491, 480, 491, 489, 471, 534, 532, 480, 474, 519, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[508, 507, 552, 507, 560, 502, 519, 498, 541, 478, 500, 535, 524, 541, 496, 544, 495, 515, 502, 477, 518, 493, 482, 473, 536, 512, 501, 549, 545, 508, 519, 477, 512, 533, 473, 533, 507, 540, 484, 534, 487, 508, 510, 526, 555, 513, 497, 500, 482, 486, 534, 523, 520, 509, 501, 504, 492, 529, 498, 523, 522, 523, 480, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 509, 506, 573, 452, 521, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 518, 470, 507, 520, 489, 522, 0, 0, 0, 0, 0, 0, 0, 0, 551, 483, 533, 558, 485, 527, 493, 520, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 496, 525, 501, 500, 516, 518, 0, 0, 0, 0, 0, 0, 0, 0, 493, 511, 498, 495, 546, 488, 490, 476, 17, 9, 5, 4, 0, 0, 0, 0, 533, 556, 505, 525, 531, 501, 497, 510, 0, 0, 0, 0, 0, 0, 0, 0, 482, 483, 523, 547, 530, 509, 504, 518, 0, 0, 0, 0, 0, 0, 0, 0, 516, 504, 462, 539, 520, 496, 502, 515, 42, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4029, 4150, 4058, 4032, 4158, 4096, 4113] avg=4096.0000 max=4158 max_rank=5 min=4029 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 534, 506, 530, 480, 519, 523, 515, 0, 0, 0, 0, 0, 0, 0, 0, 462, 558, 517, 511, 492, 515, 531, 538, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 504, 577, 545, 516, 517, 473, 0, 0, 0, 0, 0, 0, 0, 0, 486, 508, 487, 520, 526, 511, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 539, 521, 484, 467, 528, 494, 527, 18, 9, 4, 5, 0, 0, 0, 0, 479, 528, 535, 514, 505, 505, 486, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 489, 519, 521, 507, 499, 554, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 505, 514, 493, 470, 516, 479, 44, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4124, 4147, 4047, 4108, 4051, 4098, 4095] avg=4096.0000 max=4147 max_rank=2 min=4047 min_rank=3 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[459, 506, 507, 537, 471, 516, 479, 513, 0, 0, 0, 0, 0, 0, 0, 0, 524, 527, 521, 510, 539, 513, 548, 526, 0, 0, 0, 0, 0, 0, 0, 0, 538, 530, 473, 496, 503, 527, 514, 506, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 541, 521, 496, 499, 518, 537, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 483, 522, 528, 502, 468, 518, 17, 8, 4, 4, 0, 0, 0, 0, 537, 510, 546, 471, 512, 522, 537, 528, 0, 0, 0, 0, 0, 0, 0, 0, 521, 533, 472, 535, 522, 493, 519, 535, 0, 0, 0, 0, 0, 0, 0, 0, 475, 492, 457, 513, 508, 498, 488, 487, 43, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4208, 4087, 4126, 4089, 4163, 4130, 3977] avg=4096.0000 max=4208 max_rank=1 min=3977 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[495, 534, 519, 492, 457, 515, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 490, 495, 486, 552, 483, 474, 0, 0, 0, 0, 0, 0, 0, 0, 522, 530, 523, 536, 513, 489, 519, 528, 0, 0, 0, 0, 0, 0, 0, 0, 552, 547, 496, 497, 488, 531, 477, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 491, 493, 505, 540, 483, 496, 511, 16, 8, 4, 4, 0, 0, 0, 0, 546, 526, 450, 489, 490, 542, 507, 514, 0, 0, 0, 0, 0, 0, 0, 0, 481, 564, 502, 511, 531, 543, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 508, 521, 517, 520, 500, 501, 42, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 3995, 4160, 4102, 4052, 4064, 4183, 4159] avg=4096.0000 max=4183 max_rank=6 min=3995 min_rank=1 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 538, 533, 480, 503, 539, 516, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 492, 501, 502, 474, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 482, 476, 515, 519, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 522, 494, 525, 499, 536, 517, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 500, 506, 493, 492, 556, 524, 525, 483, 18, 7, 4, 4, 0, 0, 0, 0, 514, 530, 499, 523, 485, 526, 550, 520, 0, 0, 0, 0, 0, 0, 0, 0, 532, 491, 501, 524, 497, 494, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 534, 509, 529, 498, 458, 475, 511, 517, 46, 16, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4007, 4069, 4124, 4112, 4147, 4090, 4093] avg=4096.0000 max=4147 max_rank=5 min=4007 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 508, 499, 513, 475, 540, 501, 553, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 548, 557, 523, 481, 534, 471, 0, 0, 0, 0, 0, 0, 0, 0, 519, 483, 496, 515, 553, 511, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 522, 540, 505, 515, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 496, 494, 491, 497, 517, 498, 17, 8, 4, 4, 0, 0, 0, 0, 528, 510, 521, 528, 470, 514, 462, 532, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 514, 531, 520, 493, 512, 552, 0, 0, 0, 0, 0, 0, 0, 0, 500, 537, 494, 472, 536, 490, 519, 515, 44, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4134, 4101, 4091, 4023, 4065, 4147, 4122] avg=4096.0000 max=4147 max_rank=6 min=4023 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[544, 545, 541, 531, 483, 521, 542, 473, 0, 0, 0, 0, 0, 0, 0, 0, 489, 518, 520, 552, 519, 498, 475, 545, 0, 0, 0, 0, 0, 0, 0, 0, 543, 511, 496, 544, 507, 522, 488, 495, 0, 0, 0, 0, 0, 0, 0, 0, 511, 485, 491, 520, 498, 531, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 518, 527, 510, 514, 505, 534, 19, 13, 8, 8, 0, 0, 0, 0, 510, 488, 518, 523, 467, 525, 529, 486, 0, 0, 0, 0, 0, 0, 0, 0, 501, 450, 521, 495, 533, 491, 480, 491, 3, 0, 0, 0, 0, 0, 0, 0, 489, 471, 534, 532, 480, 474, 519, 494, 49, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4116, 4106, 4091, 4202, 4046, 3965, 4062] avg=4096.0000 max=4202 max_rank=4 min=3965 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16275 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 507, 552, 507, 513, 502, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 541, 474, 500, 535, 524, 541, 496, 544, 0, 0, 0, 0, 0, 0, 0, 0, 495, 515, 502, 470, 518, 493, 482, 473, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 501, 549, 527, 508, 519, 477, 0, 0, 0, 0, 0, 0, 0, 0, 512, 533, 473, 533, 507, 540, 484, 534, 18, 7, 5, 4, 0, 0, 0, 0, 487, 508, 510, 526, 533, 513, 497, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 486, 534, 523, 520, 509, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 492, 529, 498, 523, 522, 523, 480, 516, 47, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4155, 3948, 4129, 4150, 4074, 4059, 4147] avg=4096.0000 max=4155 max_rank=1 min=3948 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 3840, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[572, 477, 526, 504, 531, 510, 530, 526, 533, 495, 529, 479, 537, 514, 498, 498, 495, 492, 514, 536, 517, 527, 537, 509, 513, 515, 524, 541, 547, 501, 520, 524, 525, 514, 480, 512, 515, 477, 500, 468, 492, 497, 488, 516, 516, 521, 498, 515, 515, 509, 524, 520, 524, 524, 516, 513, 485, 500, 518, 491, 514, 521, 488, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[495, 529, 461, 526, 538, 515, 522, 533, 489, 523, 550, 533, 531, 544, 495, 514, 535, 499, 484, 478, 499, 521, 515, 475, 507, 533, 470, 511, 496, 513, 529, 527, 527, 514, 506, 528, 547, 492, 548, 492, 556, 515, 519, 535, 506, 518, 522, 493, 513, 476, 506, 522, 475, 489, 532, 540, 480, 500, 525, 525, 500, 477, 504, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[516, 479, 503, 511, 491, 510, 539, 535, 523, 497, 511, 525, 498, 507, 525, 498, 536, 498, 501, 502, 521, 525, 526, 490, 542, 514, 516, 509, 483, 533, 520, 476, 528, 522, 516, 536, 492, 520, 523, 506, 502, 498, 516, 511, 517, 489, 510, 469, 556, 519, 541, 531, 502, 514, 512, 507, 494, 506, 530, 537, 491, 525, 494, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[526, 539, 493, 522, 533, 497, 487, 500, 501, 503, 530, 522, 529, 519, 462, 519, 546, 541, 525, 542, 499, 522, 485, 532, 474, 517, 542, 519, 503, 484, 555, 491, 517, 539, 506, 474, 548, 527, 442, 515, 498, 516, 514, 464, 488, 553, 517, 535, 513, 513, 517, 528, 480, 505, 454, 521, 509, 532, 495, 506, 502, 535, 540, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[513, 529, 497, 531, 528, 529, 476, 507, 497, 521, 526, 483, 533, 496, 489, 506, 502, 492, 508, 551, 531, 487, 562, 504, 497, 570, 542, 516, 508, 524, 518, 505, 537, 491, 500, 530, 533, 519, 505, 508, 499, 549, 491, 522, 475, 483, 503, 509, 499, 500, 523, 552, 501, 491, 487, 492, 529, 522, 535, 490, 502, 480, 499, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[502, 502, 498, 536, 543, 483, 526, 520, 498, 502, 520, 492, 492, 523, 553, 461, 503, 505, 533, 549, 544, 528, 495, 499, 496, 503, 511, 493, 488, 508, 522, 483, 506, 512, 540, 544, 462, 508, 505, 535, 551, 535, 502, 527, 504, 504, 523, 497, 525, 496, 556, 494, 524, 503, 485, 528, 497, 524, 511, 539, 482, 542, 490, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[505, 519, 508, 489, 513, 519, 496, 529, 487, 552, 528, 505, 504, 518, 495, 475, 502, 532, 513, 466, 497, 499, 531, 519, 524, 501, 521, 565, 539, 505, 495, 528, 487, 522, 505, 479, 494, 499, 526, 509, 530, 515, 541, 504, 516, 511, 534, 508, 532, 537, 538, 493, 492, 535, 511, 536, 500, 497, 495, 520, 518, 493, 491, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[503, 493, 511, 489, 501, 488, 487, 507, 517, 550, 511, 519, 562, 530, 530, 512, 478, 514, 494, 506, 540, 540, 515, 512, 530, 552, 483, 502, 518, 507, 539, 498, 521, 489, 482, 530, 477, 523, 513, 500, 553, 524, 524, 502, 537, 492, 503, 492, 506, 511, 523, 509, 497, 546, 499, 488, 499, 499, 527, 511, 489, 535, 508, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[572, 477, 526, 504, 531, 510, 530, 526, 1, 0, 0, 0, 0, 0, 0, 0, 533, 495, 525, 479, 537, 514, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 492, 514, 536, 517, 527, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 513, 494, 524, 541, 547, 501, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 480, 512, 515, 477, 500, 468, 6, 4, 3, 0, 0, 0, 0, 0, 492, 497, 488, 516, 516, 521, 498, 515, 0, 0, 0, 0, 0, 0, 0, 0, 515, 509, 518, 520, 524, 524, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 485, 500, 518, 491, 514, 521, 488, 501, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4079, 4115, 4164, 4004, 4043, 4139, 4047] avg=4096.0000 max=4177 max_rank=0 min=4004 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16233 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[503, 493, 511, 489, 501, 488, 487, 507, 1, 0, 0, 0, 0, 0, 0, 0, 517, 550, 507, 519, 562, 530, 530, 512, 0, 0, 0, 0, 0, 0, 0, 0, 478, 514, 494, 506, 540, 540, 503, 512, 0, 0, 0, 0, 0, 0, 0, 0, 530, 530, 483, 502, 518, 507, 539, 498, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 482, 530, 477, 523, 513, 500, 6, 4, 3, 0, 0, 0, 0, 0, 553, 524, 524, 502, 537, 492, 503, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 511, 517, 509, 497, 546, 499, 488, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 527, 511, 489, 535, 508, 521, 22, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3980, 4227, 4087, 4107, 4048, 4127, 4073, 4119] avg=4096.0000 max=4227 max_rank=1 min=3980 min_rank=0 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 502, 498, 536, 543, 483, 526, 520, 1, 0, 0, 0, 0, 0, 0, 0, 498, 502, 516, 492, 492, 523, 553, 461, 0, 0, 0, 0, 0, 0, 0, 0, 503, 505, 533, 549, 544, 528, 483, 499, 0, 0, 0, 0, 0, 0, 0, 0, 496, 483, 511, 493, 488, 508, 522, 483, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 540, 544, 462, 508, 505, 535, 6, 4, 3, 0, 0, 0, 0, 0, 551, 535, 502, 527, 504, 504, 523, 497, 0, 0, 0, 0, 0, 0, 0, 0, 525, 496, 550, 494, 524, 503, 485, 528, 0, 0, 0, 0, 0, 0, 0, 0, 497, 524, 511, 539, 482, 542, 490, 506, 20, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4037, 4144, 3984, 4125, 4143, 4105, 4119] avg=4096.0000 max=4144 max_rank=2 min=3984 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[495, 529, 461, 526, 538, 515, 522, 533, 1, 0, 0, 0, 0, 0, 0, 0, 489, 523, 545, 533, 531, 544, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 484, 478, 499, 521, 503, 475, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 470, 511, 496, 513, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 506, 528, 547, 492, 548, 492, 6, 4, 4, 0, 0, 0, 0, 0, 556, 515, 519, 535, 506, 518, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 513, 476, 500, 522, 475, 489, 532, 540, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 525, 525, 500, 477, 504, 496, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4174, 3994, 4065, 4168, 4164, 4047, 4036] avg=4096.0000 max=4174 max_rank=1 min=3994 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 479, 503, 511, 491, 510, 539, 535, 1, 0, 0, 0, 0, 0, 0, 0, 523, 497, 507, 525, 498, 507, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 501, 502, 521, 525, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 542, 493, 516, 509, 483, 533, 520, 476, 0, 0, 0, 0, 0, 0, 0, 0, 528, 522, 516, 536, 492, 520, 523, 506, 6, 4, 3, 0, 0, 0, 0, 0, 502, 498, 516, 511, 517, 489, 510, 469, 0, 0, 0, 0, 0, 0, 0, 0, 556, 519, 535, 531, 502, 514, 512, 507, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 530, 537, 491, 525, 494, 494, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4080, 4087, 4072, 4156, 4012, 4176, 4100] avg=4096.0000 max=4176 max_rank=6 min=4012 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 529, 497, 531, 528, 529, 476, 507, 1, 0, 0, 0, 0, 0, 0, 0, 497, 521, 522, 483, 533, 496, 489, 506, 0, 0, 0, 0, 0, 0, 0, 0, 502, 492, 508, 551, 531, 487, 548, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 542, 516, 508, 524, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 537, 491, 500, 530, 533, 519, 505, 508, 6, 5, 3, 0, 0, 0, 0, 0, 499, 549, 491, 522, 475, 483, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 499, 500, 517, 552, 501, 491, 487, 492, 0, 0, 0, 0, 0, 0, 0, 0, 529, 522, 535, 490, 502, 480, 499, 534, 23, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4047, 4123, 4157, 4137, 4031, 4039, 4123] avg=4096.0000 max=4157 max_rank=3 min=4031 min_rank=5 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[526, 539, 493, 522, 533, 497, 487, 500, 1, 0, 0, 0, 0, 0, 0, 0, 501, 503, 526, 522, 529, 519, 462, 519, 0, 0, 0, 0, 0, 0, 0, 0, 546, 541, 525, 542, 499, 522, 473, 532, 0, 0, 0, 0, 0, 0, 0, 0, 474, 496, 542, 519, 503, 484, 555, 491, 0, 0, 0, 0, 0, 0, 0, 0, 517, 539, 506, 474, 548, 527, 442, 515, 6, 4, 3, 0, 0, 0, 0, 0, 498, 516, 514, 464, 488, 553, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 513, 513, 511, 528, 480, 505, 454, 521, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 495, 506, 502, 535, 540, 506, 21, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4081, 4180, 4064, 4081, 4085, 4025, 4154] avg=4096.0000 max=4180 max_rank=2 min=4025 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 519, 508, 489, 513, 519, 496, 529, 8, 0, 0, 0, 0, 0, 0, 0, 487, 552, 507, 505, 504, 518, 495, 475, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 513, 466, 497, 499, 510, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 478, 521, 565, 539, 505, 495, 528, 0, 0, 0, 0, 0, 0, 0, 0, 487, 522, 505, 479, 494, 499, 526, 509, 10, 9, 9, 0, 0, 0, 0, 0, 530, 515, 541, 504, 516, 511, 534, 508, 4, 0, 0, 0, 0, 0, 0, 0, 532, 537, 528, 493, 492, 535, 511, 536, 0, 0, 0, 0, 0, 0, 0, 0, 500, 497, 495, 520, 518, 493, 491, 521, 23, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4043, 4038, 4155, 4049, 4163, 4164, 4070] avg=4096.0000 max=4164 max_rank=6 min=4038 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[516, 478, 532, 497, 477, 509, 524, 509, 510, 523, 555, 485, 548, 539, 491, 524, 498, 501, 549, 476, 552, 506, 540, 527, 526, 496, 530, 506, 526, 516, 526, 524, 509, 518, 505, 503, 488, 512, 507, 531, 502, 509, 497, 507, 533, 487, 500, 507, 525, 503, 528, 485, 493, 533, 511, 544, 495, 503, 485, 486, 508, 543, 483, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[527, 527, 517, 484, 489, 499, 494, 514, 501, 505, 476, 530, 493, 517, 512, 531, 486, 498, 531, 490, 504, 511, 546, 549, 504, 523, 529, 505, 518, 564, 503, 509, 531, 509, 489, 539, 528, 506, 503, 539, 515, 528, 509, 468, 512, 473, 496, 522, 515, 543, 519, 483, 515, 513, 506, 523, 518, 470, 497, 566, 473, 532, 534, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[514, 501, 560, 525, 512, 546, 526, 510, 503, 524, 498, 525, 487, 528, 486, 507, 525, 494, 483, 484, 546, 540, 515, 506, 521, 501, 478, 527, 530, 486, 511, 495, 508, 537, 516, 500, 536, 512, 517, 543, 506, 526, 486, 504, 487, 529, 534, 498, 514, 487, 499, 493, 490, 518, 542, 494, 491, 476, 500, 515, 530, 515, 541, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[495, 509, 517, 512, 534, 496, 496, 511, 505, 500, 529, 537, 518, 495, 553, 483, 527, 507, 531, 525, 531, 492, 511, 520, 492, 535, 533, 510, 522, 534, 476, 491, 513, 511, 514, 513, 518, 513, 498, 537, 524, 505, 533, 495, 508, 454, 525, 515, 509, 493, 558, 499, 512, 508, 504, 526, 490, 525, 513, 499, 491, 526, 513, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[573, 497, 519, 557, 500, 509, 512, 515, 540, 518, 550, 495, 524, 481, 537, 511, 514, 538, 511, 524, 498, 529, 504, 492, 497, 501, 541, 488, 538, 531, 531, 484, 471, 525, 508, 527, 492, 469, 520, 525, 523, 515, 512, 501, 485, 482, 499, 490, 524, 481, 520, 515, 507, 538, 525, 491, 548, 470, 535, 510, 489, 522, 497, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[497, 535, 512, 543, 503, 529, 523, 555, 566, 510, 505, 504, 485, 511, 543, 477, 527, 501, 494, 493, 513, 542, 527, 510, 495, 508, 509, 504, 509, 483, 497, 514, 515, 487, 521, 520, 515, 493, 499, 475, 510, 493, 498, 523, 503, 494, 527, 508, 519, 561, 515, 536, 479, 482, 519, 510, 519, 486, 531, 511, 580, 520, 493, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[548, 548, 488, 497, 496, 549, 512, 513, 509, 484, 491, 484, 534, 481, 497, 510, 494, 508, 493, 535, 549, 515, 535, 513, 509, 479, 508, 523, 523, 493, 491, 529, 463, 516, 494, 534, 512, 561, 525, 531, 499, 493, 515, 512, 516, 529, 520, 472, 535, 503, 528, 527, 497, 524, 513, 501, 521, 505, 525, 520, 520, 520, 518, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[530, 532, 505, 523, 545, 502, 517, 486, 523, 508, 488, 475, 534, 504, 516, 513, 519, 517, 493, 522, 514, 510, 494, 531, 527, 541, 511, 551, 514, 534, 518, 496, 499, 523, 505, 487, 545, 525, 519, 518, 514, 471, 516, 526, 531, 532, 544, 504, 494, 502, 472, 502, 524, 452, 512, 499, 517, 498, 503, 497, 482, 540, 498, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 532, 505, 523, 545, 502, 517, 486, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 488, 475, 534, 504, 516, 513, 5, 0, 0, 0, 0, 0, 0, 0, 519, 517, 493, 522, 492, 510, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 541, 511, 551, 508, 534, 518, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 523, 505, 487, 545, 525, 519, 512, 0, 0, 0, 0, 0, 0, 0, 0, 514, 471, 516, 526, 531, 532, 544, 504, 32, 7, 0, 0, 0, 0, 0, 0, 494, 502, 472, 502, 524, 452, 512, 499, 6, 1, 0, 0, 0, 0, 0, 0, 517, 498, 503, 497, 482, 540, 498, 524, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4066, 4078, 4186, 4115, 4177, 3964, 4074] avg=4096.0000 max=4186 max_rank=3 min=3964 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[495, 527, 517, 484, 489, 499, 494, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 476, 530, 493, 517, 512, 531, 5, 0, 0, 0, 0, 0, 0, 0, 486, 498, 531, 490, 483, 511, 546, 549, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 529, 505, 512, 564, 503, 509, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 489, 539, 528, 506, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 515, 528, 509, 468, 512, 473, 496, 522, 32, 6, 0, 0, 0, 0, 0, 0, 515, 543, 519, 483, 515, 513, 506, 523, 6, 1, 0, 0, 0, 0, 0, 0, 518, 470, 497, 566, 473, 532, 534, 508, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4070, 4094, 4149, 4138, 4061, 4124, 4113] avg=4096.0000 max=4149 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[538, 497, 519, 557, 500, 509, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 550, 495, 524, 481, 537, 511, 5, 0, 0, 0, 0, 0, 0, 0, 514, 538, 511, 524, 478, 529, 504, 492, 0, 0, 0, 0, 0, 0, 0, 0, 497, 501, 541, 488, 531, 531, 531, 484, 0, 0, 0, 0, 0, 0, 0, 0, 471, 525, 508, 527, 492, 469, 520, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 515, 512, 501, 485, 482, 499, 490, 35, 6, 0, 0, 0, 0, 0, 0, 524, 481, 520, 515, 507, 538, 525, 491, 7, 1, 0, 0, 0, 0, 0, 0, 548, 470, 535, 510, 489, 522, 497, 493, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4161, 4090, 4104, 4031, 4048, 4109, 4078] avg=4096.0000 max=4161 max_rank=1 min=4031 min_rank=4 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[485, 478, 532, 497, 477, 509, 524, 509, 0, 0, 0, 0, 0, 0, 0, 0, 510, 523, 555, 485, 548, 539, 491, 524, 5, 0, 0, 0, 0, 0, 0, 0, 498, 501, 549, 476, 528, 506, 540, 527, 0, 0, 0, 0, 0, 0, 0, 0, 526, 496, 530, 506, 519, 516, 526, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 518, 505, 503, 488, 512, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 497, 507, 533, 487, 500, 507, 31, 7, 0, 0, 0, 0, 0, 0, 525, 503, 528, 485, 493, 533, 511, 544, 7, 1, 1, 0, 0, 0, 0, 0, 495, 503, 485, 486, 508, 543, 483, 512, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4180, 4125, 4143, 4067, 4080, 4131, 4031] avg=4096.0000 max=4180 max_rank=1 min=4011 min_rank=0 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[467, 535, 512, 543, 503, 529, 523, 555, 0, 0, 0, 0, 0, 0, 0, 0, 566, 510, 505, 504, 485, 511, 543, 477, 4, 0, 0, 0, 0, 0, 0, 0, 527, 501, 494, 493, 491, 542, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 509, 504, 503, 483, 497, 514, 0, 0, 0, 0, 0, 0, 0, 0, 515, 487, 521, 520, 515, 493, 499, 470, 0, 0, 0, 0, 0, 0, 0, 0, 510, 493, 498, 523, 503, 494, 527, 508, 30, 7, 0, 0, 0, 0, 0, 0, 519, 561, 515, 536, 479, 482, 519, 510, 6, 1, 0, 0, 0, 0, 0, 0, 519, 486, 531, 511, 580, 520, 493, 502, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4105, 4085, 4013, 4020, 4093, 4128, 4157] avg=4096.0000 max=4167 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 548, 488, 497, 496, 549, 512, 513, 0, 0, 0, 0, 0, 0, 0, 0, 509, 484, 491, 484, 534, 481, 497, 510, 5, 0, 0, 0, 0, 0, 0, 0, 494, 508, 493, 535, 525, 515, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 509, 479, 508, 523, 516, 493, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0, 463, 516, 494, 534, 512, 561, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 515, 512, 516, 529, 520, 472, 33, 7, 0, 0, 0, 0, 0, 0, 535, 503, 528, 527, 497, 524, 513, 501, 7, 1, 1, 0, 0, 0, 0, 0, 521, 505, 525, 520, 520, 520, 518, 481, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 3995, 4118, 4048, 4130, 4096, 4137, 4126] avg=4096.0000 max=4137 max_rank=6 min=3995 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[483, 501, 560, 525, 512, 546, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 498, 525, 487, 528, 486, 507, 5, 0, 0, 0, 0, 0, 0, 0, 525, 494, 483, 484, 522, 540, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 521, 501, 478, 527, 523, 486, 511, 495, 0, 0, 0, 0, 0, 0, 0, 0, 508, 537, 516, 500, 536, 512, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 486, 504, 487, 529, 534, 498, 31, 7, 0, 0, 0, 0, 0, 0, 514, 487, 499, 493, 490, 518, 542, 494, 7, 1, 1, 0, 0, 0, 0, 0, 491, 476, 500, 515, 530, 515, 541, 530, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 4063, 4069, 4042, 4163, 4108, 4046, 4114] avg=4096.0000 max=4163 max_rank=0 min=4042 min_rank=3 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[460, 509, 517, 512, 534, 496, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 505, 500, 529, 537, 518, 495, 553, 483, 8, 0, 0, 0, 0, 0, 0, 0, 527, 507, 531, 525, 496, 492, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 492, 535, 533, 510, 512, 534, 476, 491, 0, 0, 0, 0, 0, 0, 0, 0, 513, 511, 514, 513, 518, 513, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 524, 505, 533, 495, 508, 454, 525, 515, 35, 11, 0, 0, 0, 0, 0, 0, 509, 493, 558, 499, 512, 508, 504, 526, 10, 5, 5, 0, 0, 0, 0, 0, 490, 525, 513, 499, 491, 526, 513, 499, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4128, 4109, 4083, 4104, 4105, 4129, 4075] avg=4096.0000 max=4129 max_rank=6 min=4035 min_rank=0 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-26 14:57:41] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 447.6 | throughput per GPU (TFLOP/s/GPU): 275.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097711E+01 | loss scale: 1.0 | grad norm: 30.488 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[511, 520, 507, 546, 510, 491, 528, 515, 550, 515, 508, 518, 504, 489, 491, 524, 505, 514, 493, 494, 544, 518, 526, 492, 519, 520, 516, 494, 486, 512, 512, 493, 515, 530, 559, 500, 485, 510, 499, 493, 478, 545, 524, 506, 526, 519, 532, 504, 513, 525, 518, 511, 487, 521, 505, 478, 538, 511, 533, 535, 512, 477, 515, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[496, 482, 509, 505, 507, 492, 492, 527, 494, 483, 516, 528, 518, 519, 519, 519, 504, 567, 529, 522, 529, 484, 532, 507, 477, 520, 517, 521, 492, 518, 523, 539, 511, 512, 526, 497, 506, 522, 514, 490, 508, 523, 530, 522, 545, 512, 526, 510, 530, 485, 526, 512, 500, 505, 503, 485, 508, 480, 494, 521, 510, 526, 540, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[490, 498, 511, 518, 502, 507, 512, 518, 525, 525, 464, 509, 500, 493, 512, 501, 503, 506, 486, 518, 493, 527, 563, 508, 486, 480, 469, 574, 502, 468, 521, 501, 479, 509, 502, 493, 554, 508, 522, 524, 498, 508, 489, 502, 547, 541, 542, 503, 532, 527, 515, 499, 554, 532, 495, 507, 463, 548, 497, 525, 561, 529, 545, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[507, 543, 509, 540, 514, 554, 515, 514, 489, 497, 518, 510, 529, 510, 506, 497, 517, 481, 506, 484, 497, 513, 522, 524, 485, 487, 504, 477, 510, 514, 530, 524, 509, 530, 551, 518, 540, 524, 548, 538, 519, 495, 490, 486, 501, 536, 545, 534, 512, 519, 493, 534, 495, 474, 479, 495, 523, 544, 497, 502, 492, 524, 475, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[498, 481, 495, 496, 545, 514, 512, 467, 511, 514, 518, 543, 456, 553, 527, 514, 524, 511, 509, 549, 494, 523, 546, 512, 497, 546, 476, 518, 528, 540, 509, 530, 516, 518, 522, 476, 490, 521, 523, 496, 535, 489, 502, 522, 534, 501, 514, 549, 504, 505, 545, 490, 504, 488, 514, 479, 490, 510, 507, 478, 544, 525, 514, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[462, 525, 499, 556, 531, 479, 509, 489, 496, 522, 511, 493, 554, 499, 489, 519, 516, 500, 528, 505, 539, 468, 480, 520, 510, 513, 485, 535, 508, 508, 522, 593, 494, 505, 490, 486, 549, 510, 532, 500, 540, 514, 470, 559, 517, 540, 487, 517, 461, 494, 475, 526, 517, 518, 558, 525, 571, 490, 480, 495, 522, 515, 523, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[486, 514, 499, 517, 539, 500, 560, 526, 507, 522, 504, 533, 510, 524, 541, 507, 512, 501, 496, 545, 529, 531, 499, 503, 513, 536, 500, 517, 519, 523, 519, 470, 475, 564, 500, 494, 492, 498, 497, 552, 490, 529, 495, 519, 529, 511, 526, 489, 499, 483, 529, 505, 475, 495, 519, 498, 517, 510, 552, 506, 480, 521, 497, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[525, 456, 542, 532, 534, 556, 487, 544, 542, 493, 539, 512, 522, 508, 550, 500, 494, 527, 532, 506, 508, 498, 540, 513, 521, 505, 503, 522, 511, 550, 512, 487, 485, 477, 515, 512, 518, 500, 480, 501, 536, 490, 505, 503, 521, 531, 500, 557, 487, 483, 537, 485, 491, 509, 460, 551, 471, 500, 483, 507, 513, 519, 534, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[511, 520, 507, 546, 510, 491, 528, 515, 0, 0, 0, 0, 0, 0, 0, 0, 550, 515, 508, 513, 504, 489, 491, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 514, 493, 494, 544, 518, 515, 492, 0, 0, 0, 0, 0, 0, 0, 0, 519, 520, 516, 494, 486, 512, 512, 493, 5, 1, 0, 0, 0, 0, 0, 0, 515, 530, 559, 500, 485, 510, 499, 493, 7, 0, 0, 0, 0, 0, 0, 0, 478, 545, 524, 506, 489, 519, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 525, 518, 511, 487, 521, 505, 478, 37, 11, 0, 0, 0, 0, 0, 0, 538, 511, 533, 535, 512, 477, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4094, 4075, 4058, 4098, 4097, 4106, 4112] avg=4096.0000 max=4128 max_rank=0 min=4058 min_rank=3 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[496, 482, 509, 505, 507, 492, 492, 527, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 516, 523, 518, 519, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 504, 567, 529, 522, 529, 484, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 477, 520, 517, 521, 492, 518, 523, 539, 5, 1, 0, 0, 0, 0, 0, 0, 511, 512, 526, 497, 506, 522, 514, 490, 7, 0, 0, 0, 0, 0, 0, 0, 508, 523, 530, 522, 507, 512, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 530, 485, 526, 512, 500, 505, 503, 485, 38, 11, 0, 0, 0, 0, 0, 0, 508, 480, 494, 521, 510, 526, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4091, 4163, 4113, 4085, 4138, 4095, 4073] avg=4096.0000 max=4163 max_rank=2 min=4010 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[490, 498, 511, 518, 502, 507, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 525, 464, 504, 500, 493, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 503, 506, 486, 518, 493, 527, 551, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 480, 469, 574, 502, 468, 521, 501, 5, 1, 0, 0, 0, 0, 0, 0, 479, 509, 502, 493, 554, 508, 522, 524, 7, 0, 0, 0, 0, 0, 0, 0, 498, 508, 489, 502, 509, 541, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 527, 515, 499, 554, 532, 495, 507, 38, 12, 0, 0, 0, 0, 0, 0, 463, 548, 497, 525, 561, 529, 537, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4024, 4092, 4007, 4098, 4092, 4211, 4188] avg=4096.0000 max=4211 max_rank=6 min=4007 min_rank=3 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16589 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[507, 543, 509, 540, 514, 554, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 489, 497, 518, 505, 529, 510, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 517, 481, 506, 484, 497, 513, 511, 524, 0, 0, 0, 0, 0, 0, 0, 0, 485, 487, 504, 477, 510, 514, 530, 524, 5, 1, 0, 0, 0, 0, 0, 0, 509, 530, 551, 518, 540, 524, 548, 538, 6, 0, 0, 0, 0, 0, 0, 0, 519, 495, 490, 486, 466, 536, 545, 534, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 493, 534, 495, 474, 479, 495, 35, 11, 0, 0, 0, 0, 0, 0, 523, 544, 497, 502, 492, 524, 468, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4196, 4051, 4033, 4037, 4264, 4071, 4047, 4069] avg=4096.0000 max=4264 max_rank=4 min=4033 min_rank=2 imbalance_ratio=1.041016 max_load_reduction=28504 max_load_reduction_pct=0.869873 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 481, 495, 496, 545, 514, 512, 467, 0, 0, 0, 0, 0, 0, 0, 0, 511, 514, 518, 538, 456, 553, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 509, 549, 494, 523, 534, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 546, 476, 518, 528, 540, 509, 530, 5, 1, 0, 0, 0, 0, 0, 0, 516, 518, 522, 476, 490, 521, 523, 496, 7, 0, 0, 0, 0, 0, 0, 0, 535, 489, 502, 522, 497, 501, 514, 549, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 545, 490, 504, 488, 514, 479, 37, 12, 0, 0, 0, 0, 0, 0, 490, 510, 507, 478, 544, 525, 506, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4131, 4156, 4150, 4069, 4109, 4078, 4067] avg=4096.0000 max=4156 max_rank=2 min=4008 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16323 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[486, 514, 499, 517, 539, 500, 560, 526, 0, 0, 0, 0, 0, 0, 0, 0, 507, 522, 504, 528, 510, 524, 541, 507, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 496, 545, 529, 531, 488, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 536, 500, 517, 519, 523, 519, 470, 5, 1, 0, 0, 0, 0, 0, 0, 475, 564, 500, 494, 492, 498, 497, 552, 6, 0, 0, 0, 0, 0, 0, 0, 490, 529, 495, 519, 492, 511, 526, 489, 0, 0, 0, 0, 0, 0, 0, 0, 499, 483, 529, 505, 475, 495, 519, 498, 37, 11, 0, 0, 0, 0, 0, 0, 517, 510, 552, 506, 480, 521, 490, 520, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 4143, 4105, 4103, 4078, 4051, 4051, 4096] avg=4096.0000 max=4143 max_rank=1 min=4051 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16276 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[462, 525, 499, 556, 531, 479, 509, 489, 0, 0, 0, 0, 0, 0, 0, 0, 496, 522, 511, 489, 554, 499, 489, 519, 0, 0, 0, 0, 0, 0, 0, 0, 516, 500, 528, 505, 539, 468, 470, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 485, 535, 508, 508, 522, 593, 4, 1, 0, 0, 0, 0, 0, 0, 494, 505, 490, 486, 549, 510, 532, 500, 7, 0, 0, 0, 0, 0, 0, 0, 540, 514, 470, 559, 481, 540, 487, 517, 0, 0, 0, 0, 0, 0, 0, 0, 461, 494, 475, 526, 517, 518, 558, 525, 36, 10, 0, 0, 0, 0, 0, 0, 571, 490, 480, 495, 522, 515, 515, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4079, 4046, 4179, 4073, 4108, 4120, 4113] avg=4096.0000 max=4179 max_rank=3 min=4046 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 456, 542, 532, 534, 556, 487, 544, 3, 0, 0, 0, 0, 0, 0, 0, 542, 493, 539, 501, 522, 508, 550, 500, 0, 0, 0, 0, 0, 0, 0, 0, 494, 527, 532, 506, 508, 498, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 521, 505, 503, 522, 511, 550, 512, 487, 8, 2, 0, 0, 0, 0, 0, 0, 485, 477, 515, 512, 518, 500, 480, 501, 11, 4, 0, 0, 0, 0, 0, 0, 536, 490, 505, 503, 480, 531, 500, 557, 0, 0, 0, 0, 0, 0, 0, 0, 487, 483, 537, 485, 491, 509, 460, 551, 41, 16, 0, 0, 0, 0, 0, 0, 471, 500, 483, 507, 513, 519, 521, 536, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4155, 4098, 4121, 4003, 4102, 4060, 4050] avg=4096.0000 max=4179 max_rank=0 min=4003 min_rank=4 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16215 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[519, 518, 490, 484, 521, 541, 541, 494, 516, 521, 507, 515, 543, 512, 494, 501, 509, 514, 514, 505, 528, 502, 515, 489, 506, 528, 516, 511, 497, 492, 528, 495, 505, 497, 517, 495, 517, 521, 520, 516, 524, 499, 552, 533, 498, 494, 526, 556, 539, 514, 517, 490, 485, 487, 483, 500, 508, 516, 539, 521, 501, 505, 526, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[526, 538, 565, 514, 502, 476, 479, 485, 564, 508, 512, 510, 535, 520, 532, 516, 524, 554, 517, 507, 522, 502, 501, 469, 529, 480, 503, 519, 511, 542, 540, 530, 525, 500, 442, 543, 513, 535, 499, 500, 503, 512, 555, 528, 494, 532, 467, 497, 483, 509, 519, 500, 551, 528, 473, 494, 505, 548, 470, 513, 513, 496, 486, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[530, 534, 530, 569, 516, 490, 483, 515, 553, 508, 497, 482, 543, 490, 497, 496, 540, 477, 492, 512, 504, 473, 522, 492, 481, 531, 515, 540, 547, 518, 539, 516, 487, 438, 496, 473, 510, 518, 492, 523, 510, 517, 547, 542, 540, 490, 508, 497, 513, 526, 485, 480, 528, 527, 530, 520, 552, 505, 508, 488, 484, 519, 525, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[487, 503, 514, 554, 535, 522, 503, 484, 523, 517, 508, 515, 482, 486, 505, 507, 541, 515, 506, 540, 511, 498, 511, 454, 504, 545, 534, 503, 493, 502, 518, 513, 517, 502, 527, 499, 488, 523, 513, 494, 519, 548, 500, 512, 541, 498, 513, 502, 500, 501, 516, 473, 548, 501, 534, 492, 514, 511, 536, 508, 535, 519, 533, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[517, 541, 521, 499, 514, 528, 523, 500, 520, 485, 517, 519, 525, 521, 520, 518, 509, 475, 538, 522, 484, 512, 495, 464, 516, 479, 481, 514, 495, 533, 502, 506, 513, 496, 561, 538, 556, 527, 504, 512, 521, 513, 508, 508, 548, 486, 499, 482, 515, 544, 511, 513, 536, 489, 492, 483, 480, 507, 567, 532, 519, 522, 480, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[471, 498, 549, 505, 506, 512, 534, 536, 501, 509, 529, 488, 525, 527, 547, 486, 514, 514, 525, 518, 493, 528, 499, 511, 561, 528, 523, 523, 515, 503, 509, 526, 505, 479, 544, 507, 521, 518, 451, 517, 504, 498, 495, 480, 515, 477, 510, 528, 538, 517, 557, 527, 524, 498, 469, 495, 494, 537, 485, 542, 507, 471, 508, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[519, 500, 496, 516, 555, 479, 515, 573, 514, 509, 506, 480, 478, 535, 528, 490, 498, 490, 511, 525, 538, 510, 500, 522, 500, 516, 516, 506, 505, 494, 525, 526, 514, 546, 498, 492, 520, 531, 490, 483, 525, 505, 522, 507, 507, 517, 537, 489, 512, 510, 537, 555, 456, 548, 559, 517, 513, 493, 538, 482, 462, 524, 477, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[515, 546, 532, 527, 498, 501, 531, 493, 518, 471, 502, 512, 528, 528, 509, 498, 508, 508, 526, 471, 523, 547, 558, 484, 526, 534, 511, 523, 493, 543, 512, 521, 530, 491, 493, 547, 543, 491, 521, 511, 535, 485, 521, 519, 501, 526, 545, 514, 496, 498, 521, 475, 499, 523, 520, 508, 501, 502, 490, 461, 502, 488, 492, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[487, 503, 485, 554, 535, 522, 503, 484, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 508, 515, 482, 486, 505, 507, 1, 0, 0, 0, 0, 0, 0, 0, 541, 515, 506, 540, 511, 498, 511, 454, 23, 0, 0, 0, 0, 0, 0, 0, 504, 545, 534, 503, 493, 502, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 527, 499, 488, 523, 513, 494, 14, 6, 0, 0, 0, 0, 0, 0, 519, 548, 488, 512, 541, 498, 513, 502, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 516, 473, 548, 501, 534, 492, 9, 0, 0, 0, 0, 0, 0, 0, 514, 511, 536, 508, 535, 519, 533, 518, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4044, 4099, 4085, 4083, 4121, 4074, 4189] avg=4096.0000 max=4189 max_rank=7 min=4044 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[471, 498, 518, 505, 506, 512, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 501, 509, 529, 488, 525, 527, 547, 486, 1, 0, 0, 0, 0, 0, 0, 0, 514, 514, 525, 518, 493, 528, 499, 511, 24, 0, 0, 0, 0, 0, 0, 0, 561, 528, 523, 523, 515, 503, 483, 526, 0, 0, 0, 0, 0, 0, 0, 0, 505, 479, 544, 507, 521, 518, 451, 517, 14, 7, 0, 0, 0, 0, 0, 0, 504, 498, 483, 480, 515, 477, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 557, 527, 524, 498, 469, 495, 9, 0, 0, 0, 0, 0, 0, 0, 494, 537, 485, 542, 507, 471, 508, 537, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4113, 4126, 4162, 4063, 3995, 4134, 4095] avg=4096.0000 max=4162 max_rank=3 min=3995 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 541, 492, 499, 514, 528, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 517, 519, 525, 521, 520, 518, 1, 0, 0, 0, 0, 0, 0, 0, 509, 475, 538, 522, 484, 512, 495, 464, 23, 0, 0, 0, 0, 0, 0, 0, 516, 479, 481, 514, 495, 533, 476, 506, 0, 0, 0, 0, 0, 0, 0, 0, 513, 496, 561, 538, 556, 527, 504, 512, 14, 6, 0, 0, 0, 0, 0, 0, 521, 513, 496, 508, 548, 486, 499, 482, 0, 0, 0, 0, 0, 0, 0, 0, 515, 544, 511, 513, 536, 489, 492, 483, 9, 0, 0, 0, 0, 0, 0, 0, 480, 507, 567, 532, 519, 522, 480, 513, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4126, 4022, 4000, 4227, 4053, 4092, 4134] avg=4096.0000 max=4227 max_rank=4 min=4000 min_rank=3 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 538, 533, 514, 502, 476, 479, 485, 0, 0, 0, 0, 0, 0, 0, 0, 564, 508, 512, 510, 535, 520, 532, 516, 1, 0, 0, 0, 0, 0, 0, 0, 524, 554, 517, 507, 522, 502, 501, 469, 25, 0, 0, 0, 0, 0, 0, 0, 529, 480, 503, 519, 511, 542, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 442, 543, 513, 535, 499, 500, 15, 7, 0, 0, 0, 0, 0, 0, 503, 512, 541, 528, 494, 532, 467, 497, 0, 0, 0, 0, 0, 0, 0, 0, 483, 509, 519, 500, 551, 528, 473, 494, 10, 0, 0, 0, 0, 0, 0, 0, 505, 548, 470, 513, 513, 496, 486, 503, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4198, 4121, 4126, 4079, 4074, 4067, 4050] avg=4096.0000 max=4198 max_rank=1 min=4050 min_rank=7 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 518, 463, 484, 521, 541, 541, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 507, 515, 543, 512, 494, 501, 1, 0, 0, 0, 0, 0, 0, 0, 509, 514, 514, 505, 528, 502, 515, 489, 21, 0, 0, 0, 0, 0, 0, 0, 506, 528, 516, 511, 497, 492, 501, 495, 0, 0, 0, 0, 0, 0, 0, 0, 505, 497, 517, 495, 517, 521, 520, 516, 14, 6, 0, 0, 0, 0, 0, 0, 524, 499, 538, 533, 498, 494, 526, 556, 0, 0, 0, 0, 0, 0, 0, 0, 539, 514, 517, 490, 485, 487, 483, 500, 10, 0, 0, 0, 0, 0, 0, 0, 508, 516, 539, 521, 501, 505, 526, 501, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4110, 4097, 4046, 4108, 4168, 4025, 4133] avg=4096.0000 max=4168 max_rank=5 min=4025 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 534, 500, 569, 516, 490, 483, 515, 0, 0, 0, 0, 0, 0, 0, 0, 553, 508, 497, 482, 543, 490, 497, 496, 1, 0, 0, 0, 0, 0, 0, 0, 540, 477, 492, 512, 504, 473, 522, 492, 23, 0, 0, 0, 0, 0, 0, 0, 481, 531, 515, 540, 547, 518, 511, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 438, 496, 473, 510, 518, 492, 523, 15, 7, 0, 0, 0, 0, 0, 0, 510, 517, 534, 542, 540, 490, 508, 497, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 485, 480, 528, 527, 530, 520, 10, 0, 0, 0, 0, 0, 0, 0, 552, 505, 508, 488, 484, 519, 525, 558, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4067, 4035, 4159, 3959, 4138, 4119, 4154] avg=4096.0000 max=4159 max_rank=3 min=3959 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[519, 500, 460, 516, 555, 479, 515, 573, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 506, 480, 478, 535, 528, 490, 3, 0, 0, 0, 0, 0, 0, 0, 498, 490, 511, 525, 538, 510, 500, 522, 26, 0, 0, 0, 0, 0, 0, 0, 500, 516, 516, 506, 505, 494, 491, 526, 0, 0, 0, 0, 0, 0, 0, 0, 514, 546, 498, 492, 520, 531, 490, 483, 17, 10, 0, 0, 0, 0, 0, 0, 525, 505, 498, 507, 507, 517, 537, 489, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 537, 555, 456, 548, 559, 517, 14, 0, 0, 0, 0, 0, 0, 0, 513, 493, 538, 482, 462, 524, 477, 527, 17, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4043, 4120, 4054, 4101, 4085, 4208, 4040] avg=4096.0000 max=4208 max_rank=6 min=4040 min_rank=7 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 546, 502, 527, 498, 501, 531, 493, 0, 0, 0, 0, 0, 0, 0, 0, 518, 471, 502, 512, 528, 528, 509, 498, 1, 0, 0, 0, 0, 0, 0, 0, 508, 508, 526, 471, 523, 547, 558, 484, 23, 0, 0, 0, 0, 0, 0, 0, 526, 534, 511, 523, 493, 543, 485, 521, 0, 0, 0, 0, 0, 0, 0, 0, 530, 491, 493, 547, 543, 491, 521, 511, 14, 7, 0, 0, 0, 0, 0, 0, 535, 485, 509, 519, 501, 526, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 496, 498, 521, 475, 499, 523, 520, 508, 9, 0, 0, 0, 0, 0, 0, 0, 501, 502, 490, 461, 502, 488, 492, 522, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4067, 4148, 4136, 4148, 4134, 4049, 3973] avg=4096.0000 max=4148 max_rank=2 min=3973 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 3968, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[496, 513, 505, 528, 504, 478, 479, 535, 507, 504, 526, 505, 481, 503, 549, 518, 512, 508, 537, 499, 505, 540, 508, 521, 497, 582, 517, 520, 511, 493, 495, 512, 520, 518, 496, 510, 549, 513, 528, 519, 502, 497, 511, 513, 498, 541, 504, 511, 511, 506, 518, 506, 493, 492, 509, 492, 500, 539, 517, 537, 516, 532, 507, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[465, 496, 476, 503, 492, 524, 538, 531, 540, 487, 487, 472, 530, 553, 522, 518, 500, 468, 481, 529, 542, 491, 518, 528, 499, 526, 481, 544, 548, 536, 540, 520, 527, 467, 517, 475, 521, 530, 501, 518, 537, 514, 520, 481, 546, 498, 501, 485, 530, 494, 535, 504, 496, 497, 536, 539, 520, 511, 498, 506, 519, 533, 520, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[479, 517, 470, 537, 495, 527, 512, 484, 497, 569, 545, 529, 457, 525, 560, 495, 551, 520, 524, 541, 520, 493, 542, 508, 488, 555, 529, 498, 481, 558, 494, 491, 522, 493, 489, 512, 485, 495, 513, 523, 531, 462, 505, 503, 490, 479, 558, 500, 508, 504, 518, 529, 490, 515, 518, 531, 502, 536, 501, 540, 489, 508, 529, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[532, 529, 528, 505, 537, 491, 509, 522, 550, 510, 476, 512, 490, 510, 498, 548, 499, 499, 537, 514, 527, 493, 506, 506, 512, 501, 504, 527, 522, 482, 501, 500, 490, 535, 515, 470, 510, 542, 515, 495, 513, 516, 550, 514, 517, 501, 506, 526, 509, 519, 485, 517, 515, 519, 509, 503, 530, 487, 519, 540, 484, 528, 501, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[498, 534, 547, 526, 531, 475, 540, 571, 485, 497, 499, 548, 511, 547, 488, 530, 512, 519, 492, 535, 520, 508, 522, 516, 498, 497, 518, 533, 498, 507, 539, 544, 477, 498, 527, 499, 514, 512, 493, 539, 489, 521, 518, 497, 510, 533, 533, 515, 501, 534, 478, 524, 454, 507, 513, 511, 478, 483, 491, 530, 503, 505, 495, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[513, 518, 526, 526, 510, 499, 508, 528, 540, 519, 542, 504, 520, 502, 537, 474, 489, 512, 539, 522, 511, 521, 550, 540, 514, 503, 471, 507, 534, 550, 526, 501, 530, 484, 497, 534, 532, 502, 521, 474, 476, 507, 518, 487, 535, 503, 514, 541, 518, 507, 489, 482, 481, 512, 521, 504, 509, 536, 522, 524, 511, 491, 479, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[515, 468, 568, 530, 523, 506, 517, 514, 525, 516, 510, 496, 512, 512, 485, 513, 487, 571, 499, 513, 500, 513, 529, 507, 511, 551, 513, 509, 521, 535, 491, 497, 494, 525, 529, 511, 532, 532, 505, 525, 509, 510, 550, 492, 463, 515, 502, 495, 488, 484, 544, 511, 517, 492, 565, 534, 457, 500, 491, 529, 496, 511, 503, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[531, 517, 522, 506, 510, 523, 507, 494, 505, 513, 498, 479, 538, 533, 516, 519, 560, 553, 484, 532, 515, 551, 511, 520, 510, 526, 515, 475, 503, 472, 528, 530, 536, 478, 527, 481, 494, 522, 520, 514, 510, 518, 516, 526, 543, 482, 495, 518, 499, 497, 537, 508, 490, 499, 515, 514, 502, 557, 521, 482, 499, 508, 497, 467] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[496, 513, 505, 528, 504, 478, 479, 527, 0, 0, 0, 0, 0, 0, 0, 0, 507, 504, 526, 505, 481, 490, 549, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 537, 499, 505, 540, 467, 521, 0, 0, 0, 0, 0, 0, 0, 0, 497, 566, 517, 520, 511, 493, 495, 512, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 496, 510, 549, 513, 528, 519, 10, 0, 0, 0, 0, 0, 0, 0, 502, 497, 511, 513, 498, 541, 504, 511, 8, 3, 0, 0, 0, 0, 0, 0, 511, 506, 518, 506, 493, 492, 509, 492, 16, 8, 0, 0, 0, 0, 0, 0, 500, 539, 517, 537, 516, 532, 507, 475, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4080, 4089, 4111, 4163, 4088, 4051, 4156] avg=4096.0000 max=4163 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[498, 534, 547, 526, 531, 475, 540, 562, 0, 0, 0, 0, 0, 0, 0, 0, 485, 497, 499, 548, 511, 533, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 492, 535, 520, 508, 479, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 518, 533, 498, 507, 539, 544, 0, 0, 0, 0, 0, 0, 0, 0, 477, 498, 527, 499, 514, 512, 493, 539, 11, 0, 0, 0, 0, 0, 0, 0, 489, 521, 518, 497, 510, 533, 533, 515, 9, 3, 0, 0, 0, 0, 0, 0, 501, 534, 478, 524, 454, 507, 513, 511, 13, 9, 0, 0, 0, 0, 0, 0, 478, 483, 491, 530, 503, 505, 495, 501, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4213, 4091, 4081, 4121, 4070, 4128, 4044, 4020] avg=4096.0000 max=4213 max_rank=0 min=4020 min_rank=7 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[465, 496, 476, 503, 492, 524, 538, 523, 0, 0, 0, 0, 0, 0, 0, 0, 540, 487, 487, 472, 530, 538, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 500, 468, 481, 529, 542, 491, 475, 528, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 481, 544, 548, 536, 540, 520, 0, 0, 0, 0, 0, 0, 0, 0, 527, 467, 517, 475, 521, 530, 501, 518, 12, 0, 0, 0, 0, 0, 0, 0, 537, 514, 520, 481, 546, 498, 501, 485, 8, 3, 0, 0, 0, 0, 0, 0, 530, 494, 535, 504, 496, 497, 536, 539, 14, 9, 0, 0, 0, 0, 0, 0, 520, 511, 498, 506, 519, 533, 520, 507, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4094, 4014, 4180, 4068, 4093, 4154, 4148] avg=4096.0000 max=4180 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16463 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[479, 517, 470, 537, 495, 527, 512, 476, 0, 0, 0, 0, 0, 0, 0, 0, 497, 569, 545, 529, 457, 511, 560, 495, 0, 0, 0, 0, 0, 0, 0, 0, 551, 520, 524, 541, 520, 493, 498, 508, 0, 0, 0, 0, 0, 0, 0, 0, 488, 540, 529, 498, 481, 558, 494, 491, 0, 0, 0, 0, 0, 0, 0, 0, 522, 493, 489, 512, 485, 495, 513, 523, 11, 0, 0, 0, 0, 0, 0, 0, 531, 462, 505, 503, 490, 479, 558, 500, 8, 3, 0, 0, 0, 0, 0, 0, 508, 504, 518, 529, 490, 515, 518, 531, 15, 9, 0, 0, 0, 0, 0, 0, 502, 536, 501, 540, 489, 508, 529, 499, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4163, 4155, 4079, 4043, 4039, 4137, 4139] avg=4096.0000 max=4163 max_rank=1 min=4013 min_rank=0 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[532, 529, 528, 505, 537, 491, 509, 514, 0, 0, 0, 0, 0, 0, 0, 0, 550, 510, 476, 512, 490, 496, 498, 548, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 537, 514, 527, 493, 465, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 488, 504, 527, 522, 482, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 490, 535, 515, 470, 510, 542, 515, 495, 11, 0, 0, 0, 0, 0, 0, 0, 513, 516, 550, 514, 517, 501, 506, 526, 8, 3, 0, 0, 0, 0, 0, 0, 509, 519, 485, 517, 515, 519, 509, 503, 13, 8, 0, 0, 0, 0, 0, 0, 530, 487, 519, 540, 484, 528, 501, 511, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4080, 4040, 4036, 4083, 4154, 4097, 4133] avg=4096.0000 max=4154 max_rank=5 min=4036 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[513, 518, 526, 526, 510, 499, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 540, 519, 542, 504, 520, 489, 537, 474, 0, 0, 0, 0, 0, 0, 0, 0, 489, 512, 539, 522, 511, 521, 505, 540, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 471, 507, 534, 550, 526, 501, 0, 0, 0, 0, 0, 0, 0, 0, 530, 484, 497, 534, 532, 502, 521, 474, 10, 0, 0, 0, 0, 0, 0, 0, 476, 507, 518, 487, 535, 503, 514, 541, 8, 3, 0, 0, 0, 0, 0, 0, 518, 507, 489, 482, 481, 512, 521, 504, 13, 9, 0, 0, 0, 0, 0, 0, 509, 536, 522, 524, 511, 491, 479, 471, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4125, 4139, 4093, 4084, 4092, 4036, 4079] avg=4096.0000 max=4139 max_rank=2 min=4036 min_rank=6 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[531, 517, 522, 506, 510, 523, 507, 481, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 498, 479, 538, 512, 516, 519, 0, 0, 0, 0, 0, 0, 0, 0, 560, 553, 484, 532, 515, 551, 460, 520, 0, 0, 0, 0, 0, 0, 0, 0, 510, 502, 515, 475, 503, 472, 528, 530, 0, 0, 0, 0, 0, 0, 0, 0, 536, 478, 527, 481, 494, 522, 520, 514, 15, 6, 0, 0, 0, 0, 0, 0, 510, 518, 516, 526, 543, 482, 495, 518, 13, 6, 0, 0, 0, 0, 0, 0, 499, 497, 537, 508, 490, 499, 515, 514, 18, 13, 0, 0, 0, 0, 0, 0, 502, 557, 521, 482, 499, 508, 497, 467, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4080, 4175, 4035, 4093, 4127, 4090, 4071] avg=4096.0000 max=4175 max_rank=2 min=4035 min_rank=3 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 468, 568, 530, 523, 506, 517, 506, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 510, 496, 512, 498, 485, 513, 0, 0, 0, 0, 0, 0, 0, 0, 487, 571, 499, 513, 500, 513, 485, 507, 0, 0, 0, 0, 0, 0, 0, 0, 511, 536, 513, 509, 521, 535, 491, 497, 0, 0, 0, 0, 0, 0, 0, 0, 494, 525, 529, 511, 532, 532, 505, 525, 11, 0, 0, 0, 0, 0, 0, 0, 509, 510, 550, 492, 463, 515, 502, 495, 8, 3, 0, 0, 0, 0, 0, 0, 488, 484, 544, 511, 517, 492, 565, 534, 15, 9, 0, 0, 0, 0, 0, 0, 457, 500, 491, 529, 496, 511, 503, 500, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4055, 4075, 4113, 4164, 4047, 4159, 4022] avg=4096.0000 max=4164 max_rank=4 min=4022 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[533, 526, 534, 529, 520, 496, 536, 514, 490, 538, 551, 474, 497, 509, 536, 503, 488, 503, 526, 508, 485, 490, 520, 495, 522, 495, 521, 475, 498, 512, 531, 536, 544, 504, 512, 493, 536, 509, 497, 554, 519, 517, 515, 520, 498, 516, 485, 511, 510, 502, 532, 492, 452, 515, 550, 515, 495, 520, 526, 539, 516, 489, 521, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 494, 495, 519, 551, 494, 510, 476, 517, 519, 523, 521, 484, 523, 500, 519, 526, 511, 516, 501, 567, 516, 507, 504, 491, 524, 517, 520, 487, 524, 499, 504, 521, 500, 527, 523, 515, 512, 475, 501, 478, 535, 508, 501, 512, 508, 516, 517, 518, 478, 543, 522, 500, 503, 538, 497, 506, 530, 525, 530, 548, 537, 480, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[518, 522, 544, 501, 490, 444, 526, 545, 527, 508, 536, 523, 506, 492, 497, 499, 515, 514, 513, 515, 538, 465, 520, 562, 468, 523, 476, 554, 541, 500, 472, 520, 486, 525, 530, 504, 537, 507, 520, 482, 491, 520, 512, 536, 564, 542, 493, 527, 505, 527, 503, 517, 536, 484, 513, 504, 455, 505, 517, 495, 526, 509, 518, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[504, 531, 519, 505, 522, 477, 495, 501, 466, 553, 497, 522, 501, 518, 516, 502, 549, 515, 514, 491, 490, 503, 499, 539, 542, 526, 527, 503, 490, 538, 522, 502, 536, 508, 500, 497, 536, 490, 507, 473, 526, 515, 548, 526, 516, 479, 488, 545, 521, 504, 518, 558, 530, 531, 508, 538, 478, 493, 505, 524, 505, 496, 507, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[516, 526, 541, 521, 474, 531, 561, 541, 529, 478, 541, 501, 513, 488, 554, 512, 513, 509, 510, 517, 508, 471, 516, 530, 543, 490, 489, 505, 508, 522, 491, 507, 500, 544, 520, 506, 525, 527, 483, 536, 500, 480, 499, 515, 527, 479, 502, 500, 489, 531, 505, 505, 492, 531, 521, 495, 514, 475, 515, 517, 516, 523, 527, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[527, 470, 544, 546, 529, 528, 523, 499, 482, 521, 512, 505, 522, 553, 531, 525, 516, 551, 514, 518, 489, 494, 536, 514, 521, 538, 528, 534, 527, 469, 532, 481, 519, 506, 491, 540, 534, 477, 521, 483, 523, 522, 529, 540, 476, 495, 504, 500, 482, 529, 510, 494, 491, 524, 486, 484, 513, 490, 481, 498, 479, 558, 496, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[516, 528, 516, 513, 528, 544, 529, 526, 523, 490, 542, 517, 502, 540, 504, 527, 472, 508, 488, 545, 526, 506, 532, 498, 481, 477, 463, 516, 502, 538, 520, 500, 535, 521, 496, 503, 486, 519, 497, 520, 494, 505, 519, 513, 550, 532, 537, 524, 538, 489, 482, 522, 506, 467, 544, 494, 474, 546, 483, 529, 480, 479, 558, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[502, 501, 534, 508, 504, 504, 505, 527, 558, 524, 486, 478, 521, 487, 555, 502, 509, 525, 521, 525, 515, 499, 505, 498, 507, 526, 511, 514, 546, 508, 494, 538, 487, 531, 504, 508, 546, 524, 448, 553, 545, 502, 528, 533, 522, 480, 504, 538, 500, 538, 497, 523, 493, 496, 552, 498, 484, 516, 515, 483, 527, 493, 481, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[533, 526, 498, 529, 520, 496, 536, 514, 0, 0, 0, 0, 0, 0, 0, 0, 490, 538, 551, 474, 497, 509, 522, 503, 0, 0, 0, 0, 0, 0, 0, 0, 488, 503, 526, 508, 485, 490, 520, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 521, 475, 498, 512, 531, 536, 10, 0, 0, 0, 0, 0, 0, 0, 544, 504, 512, 493, 536, 509, 497, 554, 1, 0, 0, 0, 0, 0, 0, 0, 519, 517, 515, 504, 498, 516, 485, 511, 0, 0, 0, 0, 0, 0, 0, 0, 510, 502, 532, 492, 452, 515, 550, 515, 7, 4, 0, 0, 0, 0, 0, 0, 495, 520, 526, 539, 516, 489, 521, 473, 36, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4084, 4014, 4100, 4150, 4065, 4079, 4124] avg=4096.0000 max=4152 max_rank=0 min=4014 min_rank=2 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 526, 504, 521, 474, 531, 561, 541, 0, 0, 0, 0, 0, 0, 0, 0, 529, 478, 541, 501, 513, 488, 539, 512, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 510, 517, 508, 471, 516, 529, 0, 0, 0, 0, 0, 0, 0, 0, 543, 490, 489, 505, 508, 522, 491, 507, 10, 0, 0, 0, 0, 0, 0, 0, 500, 544, 520, 506, 525, 527, 483, 536, 1, 0, 0, 0, 0, 0, 0, 0, 500, 480, 499, 499, 527, 479, 502, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 531, 505, 505, 492, 531, 521, 495, 7, 5, 0, 0, 0, 0, 0, 0, 514, 475, 515, 517, 516, 523, 527, 513, 37, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4101, 4073, 4065, 4142, 3986, 4081, 4146] avg=4096.0000 max=4174 max_rank=0 min=3986 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 531, 484, 505, 522, 477, 495, 501, 0, 0, 0, 0, 0, 0, 0, 0, 466, 553, 497, 522, 501, 518, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0, 549, 515, 514, 491, 490, 503, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 542, 526, 527, 503, 490, 538, 522, 502, 10, 0, 0, 0, 0, 0, 0, 0, 536, 508, 500, 497, 536, 490, 507, 473, 1, 0, 0, 0, 0, 0, 0, 0, 526, 515, 548, 510, 516, 479, 488, 545, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 518, 558, 530, 531, 508, 538, 7, 4, 0, 0, 0, 0, 0, 0, 478, 493, 505, 524, 505, 496, 507, 483, 35, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4061, 4099, 4160, 4048, 4127, 4219, 4035] avg=4096.0000 max=4219 max_rank=6 min=4019 min_rank=0 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16429 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 522, 507, 501, 490, 444, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 536, 523, 506, 492, 484, 499, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 513, 515, 538, 465, 520, 560, 0, 0, 0, 0, 0, 0, 0, 0, 468, 523, 476, 554, 541, 500, 472, 520, 9, 0, 0, 0, 0, 0, 0, 0, 486, 525, 530, 504, 537, 507, 520, 482, 2, 0, 0, 0, 0, 0, 0, 0, 491, 520, 512, 520, 564, 542, 493, 527, 0, 0, 0, 0, 0, 0, 0, 0, 505, 527, 503, 517, 536, 484, 513, 504, 7, 4, 0, 0, 0, 0, 0, 0, 455, 505, 517, 495, 526, 509, 518, 504, 37, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4075, 4140, 4063, 4093, 4169, 4100, 4075] avg=4096.0000 max=4169 max_rank=5 min=4053 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 494, 462, 519, 551, 494, 510, 476, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 523, 521, 484, 523, 487, 519, 0, 0, 0, 0, 0, 0, 0, 0, 526, 511, 516, 501, 567, 516, 507, 503, 0, 0, 0, 0, 0, 0, 0, 0, 491, 524, 517, 520, 487, 524, 499, 504, 9, 0, 0, 0, 0, 0, 0, 0, 521, 500, 527, 523, 515, 512, 475, 501, 1, 0, 0, 0, 0, 0, 0, 0, 478, 535, 508, 487, 512, 508, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 478, 543, 522, 500, 503, 538, 497, 6, 4, 0, 0, 0, 0, 0, 0, 506, 530, 525, 530, 548, 537, 480, 480, 33, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4093, 4147, 4075, 4075, 4061, 4109, 4177] avg=4096.0000 max=4177 max_rank=7 min=4031 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 470, 507, 546, 529, 528, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 482, 521, 512, 505, 522, 553, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 551, 514, 518, 489, 494, 536, 513, 0, 0, 0, 0, 0, 0, 0, 0, 521, 538, 528, 534, 527, 469, 532, 481, 10, 0, 0, 0, 0, 0, 0, 0, 519, 506, 491, 540, 534, 477, 521, 483, 1, 0, 0, 0, 0, 0, 0, 0, 523, 522, 529, 524, 476, 495, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 529, 510, 494, 491, 524, 486, 484, 7, 4, 0, 0, 0, 0, 0, 0, 513, 490, 481, 498, 479, 558, 496, 514, 37, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4137, 4131, 4140, 4072, 4073, 4011, 4075] avg=4096.0000 max=4140 max_rank=3 min=4011 min_rank=6 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16231 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[516, 528, 477, 513, 528, 544, 529, 526, 0, 0, 0, 0, 0, 0, 0, 0, 523, 490, 542, 517, 502, 540, 480, 527, 0, 0, 0, 0, 0, 0, 0, 0, 472, 508, 488, 545, 526, 506, 532, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 477, 463, 516, 502, 538, 520, 500, 14, 0, 0, 0, 0, 0, 0, 0, 535, 521, 496, 503, 486, 519, 497, 520, 7, 2, 0, 0, 0, 0, 0, 0, 494, 505, 519, 490, 550, 532, 537, 524, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 482, 522, 506, 467, 544, 494, 10, 8, 0, 0, 0, 0, 0, 0, 474, 546, 483, 529, 480, 479, 558, 509, 39, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4121, 4068, 4011, 4086, 4151, 4060, 4110] avg=4096.0000 max=4161 max_rank=0 min=4011 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 501, 498, 508, 504, 504, 505, 527, 0, 0, 0, 0, 0, 0, 0, 0, 558, 524, 486, 478, 521, 487, 540, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 521, 525, 515, 499, 505, 497, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 511, 514, 546, 508, 494, 538, 10, 0, 0, 0, 0, 0, 0, 0, 487, 531, 504, 508, 546, 524, 448, 553, 1, 0, 0, 0, 0, 0, 0, 0, 545, 502, 528, 517, 522, 480, 504, 538, 0, 0, 0, 0, 0, 0, 0, 0, 500, 538, 497, 523, 493, 496, 552, 498, 7, 5, 0, 0, 0, 0, 0, 0, 484, 516, 515, 483, 527, 493, 481, 482, 36, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4096, 4096, 4154, 4102, 4136, 4109, 4026] avg=4096.0000 max=4154 max_rank=3 min=4026 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 3968, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 3968, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[503, 543, 481, 490, 508, 528, 484, 482, 511, 517, 513, 499, 532, 530, 514, 524, 456, 502, 511, 560, 532, 474, 541, 484, 521, 527, 478, 531, 521, 529, 452, 496, 543, 488, 503, 535, 518, 489, 524, 501, 513, 561, 540, 548, 492, 488, 522, 505, 534, 522, 522, 523, 477, 527, 511, 511, 529, 502, 457, 496, 527, 542, 520, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[514, 492, 560, 542, 528, 492, 499, 484, 518, 517, 490, 486, 519, 521, 494, 515, 539, 498, 518, 492, 502, 528, 543, 509, 518, 493, 546, 519, 540, 524, 511, 505, 527, 508, 485, 527, 509, 496, 515, 504, 515, 471, 559, 524, 496, 507, 508, 488, 477, 531, 488, 500, 465, 504, 543, 521, 548, 462, 539, 496, 519, 532, 484, 564] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[513, 534, 527, 520, 533, 504, 541, 496, 524, 523, 484, 517, 522, 528, 494, 495, 529, 499, 500, 493, 523, 508, 505, 504, 514, 521, 486, 497, 514, 544, 513, 519, 501, 524, 499, 526, 532, 486, 514, 504, 512, 506, 485, 517, 507, 498, 549, 533, 492, 518, 536, 509, 511, 499, 547, 517, 510, 477, 522, 531, 475, 482, 487, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[517, 498, 498, 538, 534, 511, 506, 476, 487, 528, 510, 524, 493, 522, 447, 519, 557, 535, 531, 522, 536, 544, 520, 506, 536, 490, 517, 526, 520, 515, 538, 514, 529, 563, 549, 495, 522, 498, 507, 492, 506, 505, 521, 505, 538, 482, 519, 488, 485, 532, 496, 517, 514, 490, 484, 505, 516, 540, 491, 516, 511, 506, 451, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[514, 518, 495, 501, 563, 528, 516, 569, 498, 526, 506, 487, 502, 505, 462, 509, 556, 519, 499, 484, 492, 486, 518, 540, 479, 471, 538, 534, 542, 502, 511, 504, 494, 503, 505, 554, 513, 534, 546, 505, 522, 486, 529, 505, 487, 518, 533, 499, 507, 544, 513, 516, 514, 511, 520, 558, 517, 498, 501, 476, 472, 507, 483, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[535, 538, 536, 518, 520, 539, 473, 479, 535, 495, 511, 482, 474, 534, 514, 521, 523, 511, 511, 490, 488, 507, 524, 547, 512, 500, 525, 540, 484, 504, 523, 511, 497, 522, 502, 488, 511, 486, 496, 522, 497, 507, 506, 513, 492, 556, 504, 479, 530, 505, 538, 499, 525, 514, 520, 516, 533, 552, 520, 515, 470, 512, 528, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[493, 480, 538, 496, 513, 522, 533, 494, 541, 545, 483, 509, 490, 476, 497, 507, 499, 493, 506, 518, 542, 496, 500, 553, 505, 484, 524, 539, 524, 513, 535, 553, 500, 508, 476, 499, 502, 475, 533, 554, 512, 505, 500, 532, 496, 453, 491, 514, 500, 531, 506, 491, 541, 529, 524, 534, 463, 530, 500, 540, 564, 548, 506, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[511, 505, 499, 493, 517, 522, 505, 545, 479, 518, 496, 496, 498, 516, 488, 501, 490, 505, 502, 543, 527, 536, 534, 523, 512, 499, 534, 501, 511, 515, 544, 496, 523, 523, 517, 530, 550, 520, 498, 518, 502, 503, 494, 521, 552, 519, 483, 506, 543, 457, 519, 526, 490, 517, 478, 490, 524, 510, 491, 525, 510, 519, 507, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 543, 481, 490, 495, 528, 484, 482, 0, 0, 0, 0, 0, 0, 0, 0, 511, 517, 513, 499, 532, 530, 514, 524, 25, 19, 0, 0, 0, 0, 0, 0, 456, 502, 511, 560, 532, 474, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 478, 510, 521, 529, 452, 496, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 503, 535, 513, 489, 524, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 561, 540, 548, 492, 488, 522, 505, 13, 2, 0, 0, 0, 0, 0, 0, 534, 522, 522, 523, 477, 527, 511, 506, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 457, 496, 527, 542, 520, 524, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4184, 4035, 4034, 4096, 4184, 4122, 4107] avg=4096.0000 max=4184 max_rank=1 min=4006 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 492, 560, 542, 514, 492, 499, 484, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 490, 486, 519, 521, 494, 515, 25, 19, 0, 0, 0, 0, 0, 0, 539, 498, 518, 492, 502, 528, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 518, 493, 546, 498, 540, 524, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 485, 527, 504, 496, 515, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 471, 559, 524, 496, 507, 508, 488, 14, 2, 0, 0, 0, 0, 0, 0, 477, 531, 488, 500, 465, 504, 543, 516, 0, 0, 0, 0, 0, 0, 0, 0, 548, 462, 539, 496, 519, 532, 484, 564, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4104, 4104, 4135, 4066, 4084, 4024, 4154] avg=4096.0000 max=4154 max_rank=7 min=4024 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16328 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[513, 534, 527, 520, 519, 504, 541, 496, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 484, 517, 522, 528, 494, 495, 23, 18, 0, 0, 0, 0, 0, 0, 529, 499, 500, 493, 523, 508, 482, 504, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 486, 477, 514, 544, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 501, 524, 499, 526, 527, 486, 514, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 506, 485, 517, 507, 498, 549, 533, 14, 2, 0, 0, 0, 0, 0, 0, 492, 518, 536, 509, 511, 499, 547, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 477, 522, 531, 475, 482, 487, 538, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4128, 4038, 4088, 4081, 4123, 4124, 4032] avg=4096.0000 max=4154 max_rank=0 min=4032 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 498, 498, 538, 520, 511, 506, 476, 0, 0, 0, 0, 0, 0, 0, 0, 487, 528, 510, 524, 493, 522, 447, 519, 24, 19, 0, 0, 0, 0, 0, 0, 557, 535, 531, 522, 536, 544, 496, 506, 0, 0, 0, 0, 0, 0, 0, 0, 536, 490, 517, 505, 520, 515, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 529, 563, 549, 495, 517, 498, 507, 492, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 521, 505, 538, 482, 519, 488, 14, 2, 0, 0, 0, 0, 0, 0, 485, 532, 496, 517, 514, 490, 484, 500, 0, 0, 0, 0, 0, 0, 0, 0, 516, 540, 491, 516, 511, 506, 451, 480, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4073, 4227, 4135, 4150, 4080, 4018, 4021] avg=4096.0000 max=4227 max_rank=2 min=4018 min_rank=6 imbalance_ratio=1.031982 max_load_reduction=28541 max_load_reduction_pct=0.871002 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[535, 538, 536, 518, 506, 539, 473, 479, 0, 0, 0, 0, 0, 0, 0, 0, 535, 495, 511, 482, 474, 534, 514, 521, 24, 19, 0, 0, 0, 0, 0, 0, 523, 511, 511, 490, 488, 507, 500, 547, 0, 0, 0, 0, 0, 0, 0, 0, 512, 500, 525, 519, 484, 504, 523, 511, 0, 0, 0, 0, 0, 0, 0, 0, 497, 522, 502, 488, 506, 486, 496, 522, 0, 0, 0, 0, 0, 0, 0, 0, 497, 507, 506, 513, 492, 556, 504, 479, 14, 2, 0, 0, 0, 0, 0, 0, 530, 505, 538, 499, 525, 514, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 552, 520, 515, 470, 512, 528, 509, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4109, 4077, 4078, 4019, 4070, 4142, 4149] avg=4096.0000 max=4149 max_rank=7 min=4019 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[493, 480, 538, 496, 499, 522, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 541, 545, 483, 509, 490, 476, 497, 507, 23, 19, 0, 0, 0, 0, 0, 0, 499, 493, 506, 518, 542, 496, 477, 553, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 524, 518, 524, 513, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 476, 499, 497, 475, 533, 554, 0, 0, 0, 0, 0, 0, 0, 0, 512, 505, 500, 532, 496, 453, 491, 514, 14, 2, 0, 0, 0, 0, 0, 0, 500, 531, 506, 491, 541, 529, 524, 529, 0, 0, 0, 0, 0, 0, 0, 0, 463, 530, 500, 540, 564, 548, 506, 510, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4090, 4084, 4156, 4042, 4019, 4151, 4171] avg=4096.0000 max=4171 max_rank=7 min=4019 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 518, 495, 501, 544, 528, 516, 569, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 506, 487, 502, 505, 462, 509, 27, 24, 0, 0, 0, 0, 0, 0, 556, 519, 499, 484, 492, 486, 491, 540, 0, 0, 0, 0, 0, 0, 0, 0, 479, 471, 538, 503, 542, 502, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 494, 503, 505, 554, 499, 534, 546, 505, 0, 0, 0, 0, 0, 0, 0, 0, 522, 486, 529, 505, 487, 518, 533, 499, 19, 7, 7, 0, 0, 0, 0, 0, 507, 544, 513, 516, 514, 511, 520, 547, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 501, 476, 472, 507, 483, 524, 11, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4046, 4067, 4050, 4140, 4112, 4172, 3996] avg=4096.0000 max=4185 max_rank=0 min=3996 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 505, 499, 493, 503, 522, 505, 545, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 496, 496, 498, 516, 488, 501, 24, 18, 0, 0, 0, 0, 0, 0, 490, 505, 502, 543, 527, 536, 510, 523, 0, 0, 0, 0, 0, 0, 0, 0, 512, 499, 534, 481, 511, 515, 544, 496, 0, 0, 0, 0, 0, 0, 0, 0, 523, 523, 517, 530, 545, 520, 498, 518, 0, 0, 0, 0, 0, 0, 0, 0, 502, 503, 494, 521, 552, 519, 483, 506, 14, 2, 0, 0, 0, 0, 0, 0, 543, 457, 519, 526, 490, 517, 478, 485, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 491, 525, 510, 519, 507, 542, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4034, 4136, 4092, 4174, 4096, 4015, 4138] avg=4096.0000 max=4174 max_rank=4 min=4015 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 3968, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[495, 529, 520, 463, 516, 552, 522, 493, 508, 546, 508, 509, 500, 500, 489, 533, 553, 498, 493, 517, 551, 511, 524, 528, 486, 526, 495, 485, 509, 500, 493, 532, 505, 499, 518, 503, 543, 481, 519, 524, 510, 510, 500, 490, 475, 513, 494, 488, 490, 505, 527, 528, 497, 480, 497, 536, 489, 532, 523, 553, 550, 570, 520, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[547, 465, 528, 516, 534, 521, 545, 525, 499, 478, 511, 485, 499, 499, 512, 502, 536, 517, 519, 500, 464, 515, 550, 505, 498, 511, 528, 497, 509, 537, 543, 494, 522, 512, 501, 498, 516, 503, 516, 500, 516, 485, 548, 505, 531, 542, 549, 542, 509, 512, 557, 532, 504, 513, 503, 506, 492, 481, 517, 474, 493, 479, 524, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[522, 534, 522, 535, 557, 515, 505, 505, 523, 493, 524, 496, 511, 518, 500, 520, 538, 534, 544, 512, 495, 514, 534, 501, 494, 503, 521, 516, 528, 474, 522, 534, 509, 423, 501, 500, 511, 492, 553, 516, 493, 560, 488, 530, 534, 496, 531, 529, 489, 468, 521, 541, 535, 495, 497, 488, 505, 494, 521, 481, 491, 521, 507, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[457, 527, 522, 517, 524, 548, 512, 494, 507, 533, 516, 527, 521, 516, 551, 510, 534, 515, 488, 493, 513, 497, 480, 512, 490, 545, 519, 517, 529, 530, 522, 508, 521, 552, 529, 533, 516, 493, 522, 487, 491, 502, 488, 490, 535, 512, 498, 453, 545, 507, 484, 481, 540, 477, 530, 508, 497, 550, 486, 542, 535, 461, 520, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[465, 494, 506, 493, 545, 522, 554, 522, 541, 510, 488, 504, 528, 540, 486, 543, 538, 513, 497, 532, 480, 517, 489, 527, 535, 517, 505, 514, 512, 476, 521, 490, 510, 491, 524, 497, 537, 524, 500, 512, 510, 495, 540, 502, 480, 521, 486, 528, 486, 527, 532, 489, 513, 534, 486, 511, 508, 503, 528, 525, 498, 506, 518, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[542, 518, 509, 513, 504, 533, 497, 508, 516, 519, 502, 494, 508, 502, 499, 496, 513, 529, 478, 522, 554, 486, 515, 502, 491, 543, 483, 496, 544, 484, 540, 530, 531, 513, 521, 499, 504, 535, 539, 536, 523, 511, 536, 482, 506, 522, 500, 463, 518, 530, 496, 517, 527, 504, 532, 521, 481, 507, 512, 502, 496, 515, 509, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[525, 530, 518, 510, 512, 508, 537, 578, 540, 495, 528, 473, 521, 507, 513, 524, 539, 500, 507, 539, 525, 487, 497, 514, 506, 507, 544, 496, 513, 501, 479, 504, 501, 496, 515, 497, 516, 486, 478, 495, 506, 450, 497, 524, 513, 517, 532, 499, 513, 525, 504, 550, 515, 493, 524, 555, 517, 534, 501, 498, 501, 511, 534, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[528, 502, 507, 479, 519, 491, 515, 555, 515, 513, 501, 532, 495, 536, 502, 517, 487, 492, 536, 525, 454, 527, 516, 510, 473, 504, 532, 507, 524, 548, 523, 507, 509, 529, 569, 517, 519, 500, 482, 522, 467, 552, 518, 511, 478, 554, 538, 535, 493, 477, 533, 471, 491, 467, 514, 510, 489, 502, 492, 522, 535, 529, 537, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[547, 465, 528, 516, 493, 521, 545, 525, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 511, 485, 499, 499, 512, 502, 4, 0, 0, 0, 0, 0, 0, 0, 529, 517, 519, 500, 464, 515, 550, 505, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 528, 497, 509, 537, 543, 494, 3, 0, 0, 0, 0, 0, 0, 0, 522, 512, 501, 498, 516, 503, 516, 500, 5, 0, 0, 0, 0, 0, 0, 0, 516, 485, 548, 505, 531, 542, 549, 542, 18, 0, 0, 0, 0, 0, 0, 0, 509, 512, 557, 532, 504, 513, 503, 506, 13, 0, 0, 0, 0, 0, 0, 0, 492, 481, 517, 474, 493, 479, 524, 497, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 3989, 4099, 4120, 4073, 4236, 4149, 3962] avg=4096.0000 max=4236 max_rank=5 min=3962 min_rank=7 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 529, 520, 463, 477, 552, 522, 493, 0, 0, 0, 0, 0, 0, 0, 0, 508, 546, 508, 509, 500, 500, 489, 533, 4, 0, 0, 0, 0, 0, 0, 0, 546, 498, 493, 517, 551, 511, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 495, 485, 509, 500, 493, 532, 3, 0, 0, 0, 0, 0, 0, 0, 505, 499, 518, 503, 543, 481, 519, 524, 5, 0, 0, 0, 0, 0, 0, 0, 510, 510, 500, 490, 475, 513, 494, 488, 17, 0, 0, 0, 0, 0, 0, 0, 490, 505, 527, 528, 497, 480, 497, 536, 13, 0, 0, 0, 0, 0, 0, 0, 489, 532, 523, 553, 550, 570, 520, 515, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4097, 4168, 4029, 4097, 3997, 4073, 4256] avg=4096.0000 max=4256 max_rank=7 min=3997 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[457, 527, 522, 517, 485, 548, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 516, 527, 521, 516, 551, 510, 4, 0, 0, 0, 0, 0, 0, 0, 527, 515, 488, 493, 513, 497, 480, 512, 0, 0, 0, 0, 0, 0, 0, 0, 490, 545, 519, 517, 529, 530, 522, 508, 3, 0, 0, 0, 0, 0, 0, 0, 521, 552, 529, 533, 516, 493, 522, 487, 5, 0, 0, 0, 0, 0, 0, 0, 491, 502, 488, 490, 535, 512, 498, 453, 17, 0, 0, 0, 0, 0, 0, 0, 545, 507, 484, 481, 540, 477, 530, 508, 13, 0, 0, 0, 0, 0, 0, 0, 497, 550, 486, 542, 535, 461, 520, 509, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4185, 4025, 4163, 4158, 3986, 4085, 4104] avg=4096.0000 max=4185 max_rank=1 min=3986 min_rank=5 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[465, 494, 506, 493, 504, 522, 554, 522, 0, 0, 0, 0, 0, 0, 0, 0, 541, 510, 488, 504, 528, 540, 486, 543, 4, 0, 0, 0, 0, 0, 0, 0, 531, 513, 497, 532, 480, 517, 489, 527, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 505, 514, 512, 476, 521, 490, 3, 0, 0, 0, 0, 0, 0, 0, 510, 491, 524, 497, 537, 524, 500, 512, 5, 0, 0, 0, 0, 0, 0, 0, 510, 495, 540, 502, 480, 521, 486, 528, 18, 0, 0, 0, 0, 0, 0, 0, 486, 527, 532, 489, 513, 534, 486, 511, 13, 0, 0, 0, 0, 0, 0, 0, 508, 503, 528, 525, 498, 506, 518, 543, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4144, 4086, 4073, 4100, 4080, 4091, 4134] avg=4096.0000 max=4144 max_rank=1 min=4060 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 534, 522, 535, 513, 515, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 524, 496, 511, 518, 500, 520, 4, 0, 0, 0, 0, 0, 0, 0, 531, 534, 544, 512, 495, 514, 534, 501, 0, 0, 0, 0, 0, 0, 0, 0, 494, 503, 521, 516, 528, 474, 522, 534, 3, 0, 0, 0, 0, 0, 0, 0, 509, 423, 501, 500, 511, 492, 553, 516, 6, 0, 0, 0, 0, 0, 0, 0, 493, 560, 488, 530, 534, 496, 531, 529, 19, 0, 0, 0, 0, 0, 0, 0, 489, 468, 521, 541, 535, 495, 497, 488, 14, 0, 0, 0, 0, 0, 0, 0, 505, 494, 521, 481, 491, 521, 507, 504, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4089, 4165, 4095, 4011, 4180, 4048, 4029] avg=4096.0000 max=4180 max_rank=5 min=4011 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[542, 518, 509, 513, 466, 533, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 516, 519, 502, 494, 508, 502, 499, 496, 4, 0, 0, 0, 0, 0, 0, 0, 507, 529, 478, 522, 554, 486, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 483, 496, 544, 484, 540, 530, 2, 0, 0, 0, 0, 0, 0, 0, 531, 513, 521, 499, 504, 535, 539, 536, 5, 0, 0, 0, 0, 0, 0, 0, 523, 511, 536, 482, 506, 522, 500, 463, 17, 0, 0, 0, 0, 0, 0, 0, 518, 530, 496, 517, 527, 504, 532, 521, 12, 0, 0, 0, 0, 0, 0, 0, 481, 507, 512, 502, 496, 515, 509, 510, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4040, 4093, 4113, 4183, 4060, 4157, 4036] avg=4096.0000 max=4183 max_rank=4 min=4036 min_rank=7 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 530, 518, 510, 455, 508, 537, 578, 0, 0, 0, 0, 0, 0, 0, 0, 540, 495, 528, 473, 521, 507, 513, 524, 8, 0, 0, 0, 0, 0, 0, 0, 522, 500, 507, 539, 525, 487, 497, 514, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 544, 496, 513, 501, 479, 504, 5, 0, 0, 0, 0, 0, 0, 0, 501, 496, 515, 497, 516, 486, 478, 495, 10, 0, 0, 0, 0, 0, 0, 0, 506, 450, 497, 524, 513, 517, 532, 499, 21, 0, 0, 0, 0, 0, 0, 0, 513, 525, 504, 550, 515, 493, 524, 555, 17, 0, 0, 0, 0, 0, 0, 0, 517, 534, 501, 498, 501, 511, 534, 494, 9, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4109, 4091, 4055, 3994, 4059, 4196, 4103] avg=4096.0000 max=4196 max_rank=6 min=3994 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 502, 507, 479, 480, 491, 515, 555, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 501, 532, 495, 536, 502, 517, 4, 0, 0, 0, 0, 0, 0, 0, 481, 492, 536, 525, 454, 527, 516, 510, 0, 0, 0, 0, 0, 0, 0, 0, 473, 504, 532, 507, 524, 548, 523, 507, 2, 0, 0, 0, 0, 0, 0, 0, 509, 529, 569, 517, 519, 500, 482, 522, 5, 0, 0, 0, 0, 0, 0, 0, 467, 552, 518, 511, 478, 554, 538, 535, 17, 0, 0, 0, 0, 0, 0, 0, 493, 477, 533, 471, 491, 467, 514, 510, 13, 0, 0, 0, 0, 0, 0, 0, 489, 502, 492, 522, 535, 529, 537, 534, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4115, 4041, 4120, 4152, 4170, 3969, 4144] avg=4096.0000 max=4170 max_rank=5 min=3969 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[513, 518, 516, 536, 533, 537, 498, 537, 507, 496, 519, 482, 474, 498, 527, 510, 521, 506, 513, 515, 534, 488, 503, 521, 545, 532, 483, 493, 513, 495, 533, 497, 465, 486, 515, 505, 527, 529, 504, 483, 519, 472, 490, 497, 516, 543, 523, 497, 486, 514, 477, 515, 530, 533, 516, 534, 529, 558, 490, 526, 543, 496, 514, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[517, 527, 503, 570, 470, 509, 529, 540, 517, 524, 507, 514, 473, 508, 501, 520, 489, 504, 544, 537, 503, 475, 494, 501, 503, 514, 480, 514, 489, 560, 528, 542, 522, 533, 500, 523, 536, 478, 522, 533, 533, 512, 528, 512, 486, 492, 505, 504, 485, 508, 515, 512, 513, 468, 519, 524, 507, 463, 510, 524, 520, 521, 524, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[501, 517, 517, 532, 478, 487, 525, 485, 490, 509, 479, 541, 504, 516, 473, 525, 517, 516, 516, 527, 574, 487, 533, 519, 509, 513, 505, 539, 511, 499, 488, 496, 524, 498, 507, 520, 541, 523, 511, 504, 501, 463, 498, 519, 547, 507, 540, 531, 499, 524, 528, 511, 500, 517, 552, 491, 518, 488, 516, 530, 524, 480, 518, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[515, 537, 489, 493, 533, 494, 511, 525, 547, 533, 521, 517, 504, 523, 492, 547, 547, 487, 536, 496, 495, 528, 514, 459, 527, 511, 468, 497, 536, 514, 487, 557, 498, 516, 513, 517, 545, 531, 503, 487, 515, 501, 522, 471, 514, 500, 487, 514, 503, 513, 543, 546, 515, 496, 493, 530, 543, 498, 517, 506, 486, 513, 486, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[531, 533, 514, 493, 522, 558, 505, 536, 519, 513, 533, 510, 518, 495, 515, 482, 496, 542, 530, 530, 479, 486, 517, 536, 533, 486, 506, 526, 526, 557, 529, 514, 518, 497, 523, 527, 515, 518, 501, 493, 490, 514, 548, 463, 513, 522, 462, 518, 507, 532, 511, 487, 471, 495, 455, 502, 518, 494, 500, 524, 504, 525, 508, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[482, 532, 547, 483, 524, 529, 529, 536, 535, 508, 495, 534, 549, 537, 525, 517, 496, 484, 502, 514, 474, 513, 511, 517, 488, 538, 514, 511, 510, 500, 491, 542, 520, 508, 517, 514, 516, 541, 498, 539, 530, 516, 467, 488, 531, 491, 499, 517, 538, 510, 474, 525, 479, 512, 479, 483, 518, 499, 543, 500, 548, 522, 495, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[515, 525, 497, 542, 523, 528, 466, 481, 520, 511, 519, 507, 539, 549, 529, 552, 501, 513, 487, 504, 499, 520, 504, 499, 482, 544, 461, 509, 539, 503, 520, 522, 503, 493, 468, 503, 533, 527, 529, 537, 506, 535, 523, 512, 536, 526, 483, 544, 483, 510, 519, 530, 505, 448, 532, 453, 519, 494, 500, 529, 497, 515, 545, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[517, 522, 533, 504, 499, 546, 491, 511, 536, 512, 495, 522, 503, 514, 494, 511, 485, 543, 533, 488, 521, 513, 531, 479, 516, 537, 505, 495, 516, 504, 558, 514, 538, 485, 487, 483, 479, 515, 548, 500, 521, 543, 528, 511, 513, 506, 505, 505, 447, 525, 501, 520, 511, 505, 504, 555, 496, 525, 514, 528, 507, 524, 480, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 493, 503, 570, 470, 509, 529, 540, 0, 0, 0, 0, 0, 0, 0, 0, 503, 524, 507, 514, 473, 508, 501, 520, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 544, 537, 503, 475, 494, 501, 12, 2, 0, 0, 0, 0, 0, 0, 503, 514, 480, 514, 489, 560, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 500, 523, 536, 478, 522, 533, 0, 0, 0, 0, 0, 0, 0, 0, 533, 512, 528, 512, 486, 492, 505, 504, 11, 6, 0, 0, 0, 0, 0, 0, 485, 508, 515, 512, 513, 468, 519, 524, 34, 8, 0, 0, 0, 0, 0, 0, 507, 463, 510, 512, 520, 521, 524, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4050, 4061, 4117, 4147, 4089, 4086, 4087] avg=4096.0000 max=4147 max_rank=4 min=4050 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 497, 547, 483, 524, 529, 529, 536, 0, 0, 0, 0, 0, 0, 0, 0, 519, 508, 495, 534, 549, 537, 525, 517, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 502, 514, 474, 513, 511, 517, 11, 2, 0, 0, 0, 0, 0, 0, 488, 538, 514, 511, 510, 500, 491, 529, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 517, 514, 516, 541, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 530, 516, 467, 488, 531, 491, 499, 517, 11, 7, 0, 0, 0, 0, 0, 0, 538, 510, 474, 525, 479, 512, 479, 483, 35, 9, 0, 0, 0, 0, 0, 0, 518, 499, 543, 489, 548, 522, 495, 484, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4184, 4024, 4081, 4153, 4057, 4044, 4098] avg=4096.0000 max=4184 max_rank=1 min=4024 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[501, 483, 517, 532, 478, 487, 525, 485, 0, 0, 0, 0, 0, 0, 0, 0, 476, 509, 479, 541, 504, 516, 473, 525, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 516, 527, 574, 487, 533, 519, 12, 2, 0, 0, 0, 0, 0, 0, 509, 513, 505, 539, 511, 499, 488, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 498, 507, 520, 541, 523, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 501, 463, 498, 519, 547, 507, 540, 531, 10, 6, 0, 0, 0, 0, 0, 0, 499, 524, 528, 511, 500, 517, 552, 491, 34, 8, 0, 0, 0, 0, 0, 0, 518, 488, 516, 518, 524, 480, 518, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4008, 4023, 4203, 4048, 4128, 4122, 4164, 4072] avg=4096.0000 max=4203 max_rank=2 min=4008 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[531, 498, 514, 493, 522, 558, 505, 536, 0, 0, 0, 0, 0, 0, 0, 0, 505, 513, 533, 510, 518, 495, 515, 482, 0, 0, 0, 0, 0, 0, 0, 0, 496, 542, 530, 530, 479, 486, 517, 536, 12, 2, 0, 0, 0, 0, 0, 0, 533, 486, 506, 526, 526, 557, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 497, 523, 527, 515, 518, 501, 493, 0, 0, 0, 0, 0, 0, 0, 0, 490, 514, 548, 463, 513, 522, 462, 518, 10, 6, 0, 0, 0, 0, 0, 0, 507, 532, 511, 487, 471, 495, 455, 502, 35, 8, 0, 0, 0, 0, 0, 0, 518, 494, 500, 512, 504, 525, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4071, 4130, 4165, 4092, 4046, 4003, 4104] avg=4096.0000 max=4165 max_rank=3 min=4003 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 484, 516, 536, 533, 537, 498, 537, 0, 0, 0, 0, 0, 0, 0, 0, 493, 496, 519, 482, 474, 498, 527, 510, 0, 0, 0, 0, 0, 0, 0, 0, 521, 506, 513, 515, 534, 488, 503, 521, 12, 2, 0, 0, 0, 0, 0, 0, 545, 532, 483, 493, 513, 495, 533, 485, 0, 0, 0, 0, 0, 0, 0, 0, 465, 486, 515, 505, 527, 529, 504, 483, 0, 0, 0, 0, 0, 0, 0, 0, 519, 472, 490, 497, 516, 543, 523, 497, 10, 6, 0, 0, 0, 0, 0, 0, 486, 514, 477, 515, 530, 533, 516, 534, 34, 8, 0, 0, 0, 0, 0, 0, 529, 558, 490, 514, 543, 496, 514, 543, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 3999, 4115, 4079, 4014, 4073, 4147, 4187] avg=4096.0000 max=4187 max_rank=7 min=3999 min_rank=1 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 502, 489, 493, 533, 494, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 533, 521, 517, 504, 523, 492, 547, 0, 0, 0, 0, 0, 0, 0, 0, 547, 487, 536, 496, 495, 528, 514, 459, 12, 2, 0, 0, 0, 0, 0, 0, 527, 511, 468, 497, 536, 514, 487, 544, 0, 0, 0, 0, 0, 0, 0, 0, 498, 516, 513, 517, 545, 531, 503, 487, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 522, 471, 514, 500, 487, 514, 11, 7, 0, 0, 0, 0, 0, 0, 503, 513, 543, 546, 515, 496, 493, 530, 35, 9, 0, 0, 0, 0, 0, 0, 543, 498, 517, 494, 486, 513, 486, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4168, 4076, 4084, 4110, 4042, 4183, 4043] avg=4096.0000 max=4183 max_rank=6 min=4042 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 488, 497, 542, 523, 528, 466, 481, 0, 0, 0, 0, 0, 0, 0, 0, 496, 511, 519, 507, 539, 549, 529, 552, 0, 0, 0, 0, 0, 0, 0, 0, 501, 513, 487, 504, 499, 520, 504, 499, 16, 5, 4, 0, 0, 0, 0, 0, 482, 544, 461, 509, 539, 503, 520, 503, 0, 0, 0, 0, 0, 0, 0, 0, 503, 493, 468, 503, 529, 527, 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 523, 512, 536, 526, 483, 544, 14, 11, 0, 0, 0, 0, 0, 0, 483, 510, 519, 530, 505, 448, 532, 453, 37, 13, 0, 0, 0, 0, 0, 0, 519, 494, 500, 513, 497, 515, 545, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4202, 4052, 4061, 4089, 4190, 4030, 4104] avg=4096.0000 max=4202 max_rank=1 min=4030 min_rank=6 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 488, 533, 504, 499, 546, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 520, 512, 495, 522, 503, 514, 494, 511, 0, 0, 0, 0, 0, 0, 0, 0, 485, 543, 533, 488, 521, 513, 531, 479, 12, 2, 0, 0, 0, 0, 0, 0, 516, 537, 505, 495, 516, 504, 558, 502, 0, 0, 0, 0, 0, 0, 0, 0, 538, 485, 487, 483, 479, 515, 548, 500, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 528, 511, 513, 506, 505, 505, 10, 7, 0, 0, 0, 0, 0, 0, 447, 525, 501, 520, 511, 505, 504, 555, 34, 9, 0, 0, 0, 0, 0, 0, 496, 525, 514, 516, 507, 524, 480, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4071, 4107, 4133, 4035, 4149, 4111, 4073] avg=4096.0000 max=4149 max_rank=5 min=4035 min_rank=4 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[517, 541, 505, 561, 513, 535, 500, 542, 545, 509, 503, 528, 543, 522, 506, 486, 516, 494, 514, 487, 467, 513, 511, 512, 500, 492, 519, 539, 537, 538, 492, 482, 511, 490, 548, 524, 484, 519, 539, 479, 514, 502, 535, 518, 504, 530, 488, 562, 543, 471, 519, 486, 506, 494, 516, 499, 492, 479, 519, 480, 488, 514, 524, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[521, 559, 528, 513, 530, 514, 480, 491, 548, 515, 529, 556, 530, 544, 495, 526, 488, 490, 551, 515, 509, 501, 492, 494, 503, 539, 509, 517, 478, 523, 493, 500, 497, 562, 507, 517, 505, 553, 488, 473, 517, 525, 496, 507, 531, 485, 514, 466, 494, 492, 499, 490, 533, 486, 522, 530, 508, 564, 517, 495, 521, 479, 506, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[487, 527, 522, 488, 509, 508, 497, 508, 513, 509, 525, 543, 484, 557, 527, 500, 498, 490, 494, 506, 516, 506, 502, 454, 527, 534, 503, 531, 508, 541, 528, 529, 498, 515, 541, 518, 521, 489, 516, 507, 508, 489, 528, 502, 502, 508, 499, 522, 519, 492, 516, 503, 493, 502, 525, 522, 480, 531, 520, 525, 516, 545, 532, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[499, 543, 500, 520, 514, 510, 490, 502, 518, 488, 505, 511, 481, 539, 487, 494, 500, 534, 527, 546, 528, 531, 485, 473, 517, 517, 506, 479, 493, 513, 486, 542, 520, 525, 517, 514, 543, 519, 521, 529, 502, 488, 535, 516, 495, 501, 499, 533, 513, 534, 532, 513, 472, 496, 514, 522, 537, 476, 539, 506, 530, 548, 507, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[508, 508, 545, 477, 543, 535, 495, 500, 497, 512, 519, 520, 502, 503, 482, 515, 499, 553, 532, 503, 473, 471, 534, 537, 479, 514, 494, 521, 457, 494, 529, 493, 557, 513, 497, 562, 542, 480, 469, 538, 553, 531, 567, 553, 517, 502, 496, 523, 545, 499, 498, 492, 501, 496, 490, 547, 528, 511, 481, 508, 519, 497, 472, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[474, 504, 530, 502, 493, 510, 502, 520, 521, 481, 504, 562, 548, 544, 509, 507, 514, 486, 513, 538, 510, 542, 515, 515, 500, 526, 507, 543, 487, 502, 523, 509, 515, 489, 471, 497, 522, 529, 534, 515, 516, 558, 511, 496, 492, 539, 490, 491, 532, 534, 503, 536, 513, 487, 529, 535, 491, 505, 507, 490, 485, 534, 507, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[523, 529, 488, 545, 532, 512, 528, 517, 503, 537, 454, 548, 537, 508, 524, 510, 473, 526, 515, 485, 496, 513, 499, 509, 492, 519, 513, 508, 516, 505, 506, 527, 521, 538, 517, 503, 462, 521, 539, 523, 490, 529, 513, 539, 494, 522, 502, 541, 506, 474, 501, 581, 508, 513, 475, 478, 492, 497, 526, 492, 531, 536, 485, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[497, 503, 507, 511, 531, 511, 515, 496, 517, 510, 516, 548, 519, 521, 507, 497, 520, 525, 516, 528, 526, 515, 527, 516, 573, 549, 534, 533, 527, 512, 496, 511, 498, 514, 493, 504, 539, 483, 487, 501, 482, 530, 538, 498, 519, 483, 466, 515, 504, 492, 512, 491, 514, 495, 519, 543, 506, 513, 465, 555, 536, 496, 476, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 531, 500, 520, 514, 510, 490, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 488, 505, 479, 481, 539, 487, 494, 0, 0, 0, 0, 0, 0, 0, 0, 500, 534, 527, 546, 528, 531, 485, 473, 23, 0, 0, 0, 0, 0, 0, 0, 517, 512, 506, 479, 493, 513, 486, 542, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 517, 514, 543, 519, 521, 529, 0, 0, 0, 0, 0, 0, 0, 0, 502, 488, 529, 516, 495, 501, 499, 533, 0, 0, 0, 0, 0, 0, 0, 0, 513, 534, 532, 513, 472, 496, 514, 522, 10, 6, 5, 0, 0, 0, 0, 0, 537, 476, 539, 506, 530, 548, 507, 494, 12, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 3991, 4147, 4048, 4177, 4063, 4117, 4159] avg=4096.0000 max=4177 max_rank=4 min=3991 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[487, 515, 522, 488, 509, 508, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 525, 508, 484, 557, 527, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 494, 506, 516, 506, 502, 454, 25, 0, 0, 0, 0, 0, 0, 0, 527, 529, 503, 531, 508, 541, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 498, 504, 541, 518, 521, 489, 516, 507, 0, 0, 0, 0, 0, 0, 0, 0, 508, 489, 522, 502, 502, 508, 499, 522, 0, 0, 0, 0, 0, 0, 0, 0, 519, 492, 516, 503, 493, 502, 525, 522, 10, 6, 5, 0, 0, 0, 0, 0, 480, 531, 520, 525, 516, 545, 532, 513, 12, 10, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4123, 3991, 4196, 4094, 4052, 4093, 4185] avg=4096.0000 max=4196 max_rank=3 min=3991 min_rank=2 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[521, 547, 528, 513, 530, 514, 480, 491, 0, 0, 0, 0, 0, 0, 0, 0, 548, 515, 529, 521, 530, 544, 495, 526, 0, 0, 0, 0, 0, 0, 0, 0, 488, 490, 551, 515, 509, 501, 492, 494, 25, 0, 0, 0, 0, 0, 0, 0, 503, 534, 509, 517, 478, 523, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 497, 550, 507, 517, 505, 553, 488, 473, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 491, 507, 531, 485, 514, 466, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 499, 490, 533, 486, 522, 530, 11, 5, 5, 0, 0, 0, 0, 0, 508, 564, 517, 495, 521, 479, 506, 508, 12, 10, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4208, 4065, 4057, 4090, 4036, 4067, 4121] avg=4096.0000 max=4208 max_rank=1 min=4036 min_rank=5 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 529, 505, 561, 513, 535, 500, 542, 0, 0, 0, 0, 0, 0, 0, 0, 545, 509, 503, 495, 543, 522, 506, 486, 0, 0, 0, 0, 0, 0, 0, 0, 516, 494, 514, 487, 467, 513, 511, 512, 24, 0, 0, 0, 0, 0, 0, 0, 500, 487, 519, 539, 537, 538, 492, 482, 0, 0, 0, 0, 0, 0, 0, 0, 511, 480, 548, 524, 484, 519, 539, 479, 0, 0, 0, 0, 0, 0, 0, 0, 514, 502, 529, 518, 504, 530, 488, 562, 0, 0, 0, 0, 0, 0, 0, 0, 543, 471, 519, 486, 506, 494, 516, 499, 9, 6, 5, 0, 0, 0, 0, 0, 492, 479, 519, 480, 488, 514, 524, 522, 12, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4202, 4109, 4038, 4094, 4084, 4147, 4054, 4040] avg=4096.0000 max=4202 max_rank=0 min=4038 min_rank=2 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 497, 545, 477, 543, 535, 495, 500, 0, 0, 0, 0, 0, 0, 0, 0, 497, 512, 519, 487, 502, 503, 482, 515, 0, 0, 0, 0, 0, 0, 0, 0, 499, 553, 532, 503, 473, 471, 534, 537, 24, 0, 0, 0, 0, 0, 0, 0, 479, 509, 494, 521, 457, 494, 529, 493, 0, 0, 0, 0, 0, 0, 0, 0, 557, 502, 497, 562, 542, 480, 469, 538, 0, 0, 0, 0, 0, 0, 0, 0, 553, 531, 561, 553, 517, 502, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 545, 499, 498, 492, 501, 496, 490, 547, 10, 6, 5, 0, 0, 0, 0, 0, 528, 511, 481, 508, 519, 497, 472, 540, 11, 9, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4017, 4126, 3976, 4147, 4236, 4089, 4077] avg=4096.0000 max=4236 max_rank=5 min=3976 min_rank=3 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16549 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[474, 493, 530, 502, 493, 510, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 521, 481, 504, 526, 548, 544, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 514, 486, 513, 538, 510, 542, 515, 515, 26, 0, 0, 0, 0, 0, 0, 0, 500, 521, 507, 543, 487, 502, 523, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 479, 471, 497, 522, 529, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 516, 558, 506, 496, 492, 539, 490, 491, 0, 0, 0, 0, 0, 0, 0, 0, 532, 534, 503, 536, 513, 487, 529, 535, 9, 5, 5, 0, 0, 0, 0, 0, 491, 505, 507, 490, 485, 534, 507, 474, 11, 10, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4140, 4159, 4092, 4062, 4088, 4188, 4015] avg=4096.0000 max=4188 max_rank=6 min=4015 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 492, 507, 511, 531, 511, 515, 496, 0, 0, 0, 0, 0, 0, 0, 0, 517, 510, 516, 513, 519, 521, 507, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 516, 528, 526, 515, 527, 516, 25, 0, 0, 0, 0, 0, 0, 0, 573, 543, 534, 533, 527, 512, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 493, 504, 539, 483, 487, 501, 0, 0, 0, 0, 0, 0, 0, 0, 482, 530, 532, 498, 519, 483, 466, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 492, 512, 491, 514, 495, 519, 543, 10, 6, 6, 0, 0, 0, 0, 0, 506, 513, 465, 555, 536, 496, 476, 487, 11, 10, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4100, 4198, 4229, 4008, 4025, 4092, 4056] avg=4096.0000 max=4229 max_rank=3 min=4008 min_rank=4 imbalance_ratio=1.032471 max_load_reduction=28539 max_load_reduction_pct=0.870941 moved_to_echo=16181 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 513, 488, 545, 532, 512, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 537, 454, 507, 537, 508, 524, 510, 0, 0, 0, 0, 0, 0, 0, 0, 473, 526, 515, 485, 496, 513, 499, 509, 28, 0, 0, 0, 0, 0, 0, 0, 492, 509, 513, 508, 516, 505, 506, 527, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 517, 503, 462, 521, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 490, 529, 504, 539, 494, 522, 502, 541, 0, 0, 0, 0, 0, 0, 0, 0, 506, 474, 501, 581, 508, 513, 475, 478, 13, 9, 10, 0, 0, 0, 0, 0, 492, 497, 526, 492, 531, 536, 485, 522, 16, 13, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4080, 4044, 4076, 4106, 4121, 4068, 4115] avg=4096.0000 max=4158 max_rank=0 min=4044 min_rank=2 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4352, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[522, 535, 545, 493, 522, 487, 512, 511, 487, 548, 519, 509, 513, 512, 518, 520, 501, 496, 497, 481, 542, 504, 535, 518, 565, 530, 553, 455, 521, 544, 519, 539, 477, 534, 496, 539, 522, 517, 513, 473, 514, 492, 487, 499, 502, 509, 485, 526, 512, 541, 494, 472, 510, 531, 471, 496, 484, 517, 506, 505, 551, 509, 532, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[534, 507, 542, 533, 444, 527, 511, 490, 517, 494, 503, 495, 495, 481, 515, 514, 522, 528, 452, 536, 501, 517, 518, 482, 536, 548, 513, 490, 520, 524, 497, 531, 528, 563, 493, 533, 498, 511, 547, 506, 509, 502, 525, 541, 486, 497, 497, 498, 532, 507, 507, 518, 541, 435, 529, 497, 532, 531, 518, 536, 497, 484, 526, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[567, 494, 506, 493, 540, 505, 502, 511, 517, 487, 538, 525, 499, 563, 469, 488, 502, 481, 540, 530, 494, 464, 517, 493, 504, 518, 545, 534, 497, 482, 496, 503, 558, 565, 526, 514, 508, 513, 524, 517, 515, 533, 492, 552, 531, 503, 505, 510, 548, 494, 502, 500, 473, 539, 498, 511, 520, 517, 536, 516, 504, 482, 484, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[488, 516, 462, 521, 494, 536, 530, 511, 514, 488, 539, 504, 465, 490, 547, 505, 520, 494, 561, 529, 514, 532, 520, 502, 508, 530, 538, 505, 535, 511, 524, 485, 546, 483, 526, 529, 493, 507, 538, 521, 489, 500, 507, 496, 515, 488, 481, 517, 529, 501, 502, 509, 498, 522, 480, 527, 531, 528, 489, 530, 521, 520, 523, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[515, 513, 481, 505, 488, 490, 503, 530, 496, 527, 490, 517, 500, 515, 549, 502, 506, 487, 470, 516, 481, 519, 505, 520, 552, 560, 505, 474, 522, 505, 500, 505, 528, 531, 527, 541, 519, 487, 498, 563, 518, 502, 515, 533, 525, 538, 498, 509, 514, 520, 537, 519, 468, 459, 463, 514, 527, 500, 523, 552, 522, 535, 502, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[480, 524, 491, 486, 552, 517, 508, 515, 509, 490, 513, 517, 477, 521, 552, 510, 494, 553, 521, 479, 565, 523, 505, 516, 544, 518, 479, 490, 482, 495, 503, 534, 495, 500, 496, 513, 556, 517, 551, 525, 483, 503, 519, 544, 516, 519, 501, 517, 542, 553, 502, 504, 513, 482, 533, 482, 494, 543, 528, 512, 474, 494, 499, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[527, 500, 508, 507, 502, 521, 521, 514, 509, 479, 522, 486, 512, 467, 531, 502, 492, 522, 514, 504, 527, 506, 531, 494, 519, 510, 540, 505, 535, 517, 512, 523, 524, 502, 523, 522, 522, 507, 500, 496, 564, 528, 507, 513, 491, 507, 486, 527, 560, 513, 481, 505, 505, 529, 529, 543, 490, 503, 521, 494, 518, 492, 511, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[543, 496, 526, 526, 491, 525, 511, 440, 528, 483, 506, 535, 494, 539, 533, 545, 497, 502, 481, 522, 525, 499, 497, 475, 523, 527, 440, 525, 480, 488, 549, 489, 521, 519, 522, 496, 498, 499, 530, 491, 533, 544, 526, 544, 545, 532, 460, 495, 535, 524, 521, 514, 506, 502, 493, 562, 493, 526, 534, 503, 526, 485, 525, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[522, 535, 545, 493, 522, 487, 512, 511, 11, 4, 0, 0, 0, 0, 0, 0, 487, 548, 519, 509, 513, 512, 518, 520, 16, 0, 0, 0, 0, 0, 0, 0, 501, 496, 497, 481, 542, 504, 535, 518, 24, 0, 0, 0, 0, 0, 0, 0, 541, 530, 553, 455, 521, 544, 519, 539, 0, 0, 0, 0, 0, 0, 0, 0, 477, 534, 496, 539, 522, 517, 469, 473, 0, 0, 0, 0, 0, 0, 0, 0, 514, 492, 487, 499, 502, 509, 485, 526, 2, 0, 0, 0, 0, 0, 0, 0, 512, 541, 494, 472, 510, 531, 471, 496, 13, 1, 0, 0, 0, 0, 0, 0, 484, 514, 506, 505, 551, 509, 532, 499, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4142, 4098, 4202, 4027, 4016, 4041, 4100] avg=4096.0000 max=4202 max_rank=3 min=4016 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[567, 494, 506, 493, 540, 505, 502, 511, 9, 4, 0, 0, 0, 0, 0, 0, 517, 487, 538, 525, 499, 563, 469, 488, 16, 0, 0, 0, 0, 0, 0, 0, 502, 481, 540, 530, 494, 464, 517, 493, 24, 0, 0, 0, 0, 0, 0, 0, 484, 518, 545, 534, 497, 482, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 558, 565, 526, 514, 508, 513, 480, 517, 0, 0, 0, 0, 0, 0, 0, 0, 515, 533, 492, 552, 531, 503, 505, 510, 2, 0, 0, 0, 0, 0, 0, 0, 548, 494, 502, 500, 473, 539, 498, 511, 11, 1, 0, 0, 0, 0, 0, 0, 520, 514, 536, 516, 504, 482, 484, 474, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4102, 4045, 4059, 4181, 4143, 4077, 4030] avg=4096.0000 max=4181 max_rank=4 min=4030 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 513, 481, 505, 488, 490, 503, 530, 10, 4, 0, 0, 0, 0, 0, 0, 496, 527, 490, 517, 500, 515, 549, 502, 15, 0, 0, 0, 0, 0, 0, 0, 506, 487, 470, 516, 481, 519, 505, 520, 23, 0, 0, 0, 0, 0, 0, 0, 530, 560, 505, 474, 522, 505, 500, 505, 0, 0, 0, 0, 0, 0, 0, 0, 528, 531, 527, 541, 519, 487, 456, 563, 0, 0, 0, 0, 0, 0, 0, 0, 518, 502, 515, 533, 525, 538, 498, 509, 2, 0, 0, 0, 0, 0, 0, 0, 514, 520, 537, 519, 468, 459, 463, 514, 12, 1, 0, 0, 0, 0, 0, 0, 527, 497, 523, 552, 522, 535, 502, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4111, 4027, 4101, 4152, 4140, 4007, 4191] avg=4096.0000 max=4191 max_rank=7 min=4007 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 507, 542, 533, 444, 527, 511, 490, 10, 4, 0, 0, 0, 0, 0, 0, 517, 494, 503, 495, 495, 481, 515, 514, 17, 0, 0, 0, 0, 0, 0, 0, 522, 528, 452, 536, 501, 517, 518, 482, 25, 0, 0, 0, 0, 0, 0, 0, 514, 548, 513, 490, 520, 524, 497, 531, 0, 0, 0, 0, 0, 0, 0, 0, 528, 563, 493, 533, 498, 511, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 509, 502, 525, 541, 486, 497, 497, 498, 2, 0, 0, 0, 0, 0, 0, 0, 532, 507, 507, 518, 541, 435, 529, 497, 12, 2, 0, 0, 0, 0, 0, 0, 532, 527, 518, 536, 497, 484, 526, 527, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4031, 4081, 4137, 4133, 4057, 4080, 4147] avg=4096.0000 max=4147 max_rank=7 min=4031 min_rank=1 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 516, 462, 521, 494, 536, 530, 511, 9, 4, 0, 0, 0, 0, 0, 0, 514, 488, 539, 504, 465, 490, 547, 505, 17, 0, 0, 0, 0, 0, 0, 0, 520, 494, 561, 529, 514, 532, 520, 502, 25, 0, 0, 0, 0, 0, 0, 0, 488, 530, 538, 505, 535, 511, 524, 485, 0, 0, 0, 0, 0, 0, 0, 0, 546, 483, 526, 529, 493, 507, 492, 521, 0, 0, 0, 0, 0, 0, 0, 0, 489, 500, 507, 496, 515, 488, 481, 517, 2, 0, 0, 0, 0, 0, 0, 0, 529, 501, 502, 509, 498, 522, 480, 527, 11, 2, 0, 0, 0, 0, 0, 0, 531, 524, 489, 530, 521, 520, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4069, 4197, 4116, 4097, 3995, 4081, 4142] avg=4096.0000 max=4197 max_rank=2 min=3995 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[543, 496, 526, 526, 491, 525, 511, 440, 10, 4, 0, 0, 0, 0, 0, 0, 528, 483, 506, 535, 494, 539, 533, 545, 16, 0, 0, 0, 0, 0, 0, 0, 497, 502, 481, 522, 525, 499, 497, 475, 24, 0, 0, 0, 0, 0, 0, 0, 501, 527, 440, 525, 480, 488, 549, 489, 0, 0, 0, 0, 0, 0, 0, 0, 521, 519, 522, 496, 498, 499, 486, 491, 0, 0, 0, 0, 0, 0, 0, 0, 533, 544, 526, 544, 545, 532, 460, 495, 2, 0, 0, 0, 0, 0, 0, 0, 535, 524, 521, 514, 506, 502, 493, 562, 12, 2, 0, 0, 0, 0, 0, 0, 493, 522, 534, 503, 526, 485, 525, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4179, 4022, 3999, 4032, 4181, 4171, 4112] avg=4096.0000 max=4181 max_rank=5 min=3999 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 500, 508, 507, 502, 521, 521, 514, 14, 10, 0, 0, 0, 0, 0, 0, 509, 479, 522, 486, 512, 467, 531, 502, 20, 0, 0, 0, 0, 0, 0, 0, 492, 522, 514, 504, 527, 506, 531, 494, 27, 0, 0, 0, 0, 0, 0, 0, 489, 510, 540, 505, 535, 517, 512, 523, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 523, 522, 522, 507, 443, 496, 0, 0, 0, 0, 0, 0, 0, 0, 564, 528, 507, 513, 491, 507, 486, 527, 9, 0, 0, 0, 0, 0, 0, 0, 560, 513, 481, 505, 505, 529, 529, 543, 16, 5, 0, 0, 0, 0, 0, 0, 490, 489, 521, 494, 518, 492, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4028, 4117, 4131, 4039, 4132, 4186, 4011] avg=4096.0000 max=4186 max_rank=6 min=4011 min_rank=7 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[480, 524, 491, 486, 552, 517, 508, 515, 10, 4, 0, 0, 0, 0, 0, 0, 509, 490, 513, 517, 477, 521, 552, 510, 17, 0, 0, 0, 0, 0, 0, 0, 494, 553, 521, 479, 565, 523, 505, 516, 25, 0, 0, 0, 0, 0, 0, 0, 522, 518, 479, 490, 482, 495, 503, 534, 0, 0, 0, 0, 0, 0, 0, 0, 495, 500, 496, 513, 556, 517, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 483, 503, 519, 544, 516, 519, 501, 517, 2, 0, 0, 0, 0, 0, 0, 0, 542, 553, 502, 504, 513, 482, 533, 482, 12, 2, 0, 0, 0, 0, 0, 0, 494, 539, 528, 512, 474, 494, 499, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4106, 4181, 4023, 4107, 4104, 4125, 4035] avg=4096.0000 max=4181 max_rank=2 min=4023 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4352, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3840, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[519, 481, 511, 477, 544, 540, 486, 519, 519, 533, 505, 508, 497, 497, 525, 498, 491, 537, 522, 502, 509, 504, 496, 531, 522, 529, 515, 515, 489, 473, 510, 532, 539, 506, 502, 514, 500, 538, 525, 551, 531, 508, 544, 505, 512, 484, 538, 525, 498, 529, 469, 500, 482, 527, 507, 530, 517, 487, 489, 479, 525, 516, 522, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[488, 519, 484, 501, 512, 495, 545, 514, 511, 522, 497, 507, 558, 513, 515, 528, 526, 492, 549, 522, 488, 463, 497, 508, 498, 488, 522, 526, 524, 513, 539, 500, 545, 519, 520, 513, 509, 508, 493, 518, 469, 504, 550, 505, 514, 525, 513, 481, 519, 525, 523, 524, 499, 495, 532, 526, 486, 519, 511, 488, 526, 510, 511, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[510, 496, 515, 515, 509, 516, 503, 499, 482, 505, 492, 523, 509, 528, 529, 539, 527, 502, 529, 507, 526, 493, 512, 504, 535, 525, 496, 558, 483, 517, 464, 490, 496, 547, 513, 528, 487, 527, 513, 501, 469, 505, 533, 501, 499, 485, 497, 512, 540, 529, 516, 549, 505, 500, 467, 538, 521, 503, 547, 526, 498, 530, 517, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[540, 544, 472, 477, 513, 558, 546, 465, 526, 484, 535, 517, 493, 524, 512, 534, 512, 477, 483, 503, 481, 533, 498, 502, 494, 511, 518, 498, 470, 525, 576, 535, 511, 489, 479, 500, 512, 520, 500, 484, 518, 496, 501, 514, 530, 552, 515, 524, 537, 507, 526, 512, 475, 528, 541, 497, 505, 553, 515, 528, 502, 496, 528, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[522, 508, 554, 493, 560, 551, 462, 501, 547, 492, 503, 549, 484, 504, 482, 534, 518, 493, 513, 511, 541, 511, 510, 531, 529, 490, 518, 502, 510, 481, 530, 529, 494, 518, 468, 567, 484, 542, 531, 497, 499, 521, 542, 521, 495, 502, 499, 534, 515, 511, 511, 496, 483, 523, 486, 525, 517, 492, 481, 498, 543, 502, 488, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[534, 506, 474, 516, 520, 493, 477, 504, 532, 497, 494, 529, 509, 514, 506, 525, 518, 525, 488, 541, 521, 507, 482, 527, 508, 514, 522, 532, 480, 542, 477, 538, 521, 505, 486, 513, 531, 521, 526, 575, 515, 511, 535, 514, 538, 511, 498, 510, 521, 489, 498, 520, 496, 519, 518, 517, 507, 506, 486, 478, 509, 522, 509, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[552, 522, 520, 537, 533, 521, 520, 532, 497, 530, 522, 496, 516, 514, 476, 493, 506, 492, 537, 527, 532, 514, 540, 491, 498, 535, 545, 530, 497, 518, 459, 505, 535, 502, 513, 494, 483, 482, 520, 460, 482, 513, 521, 502, 530, 507, 486, 506, 514, 490, 515, 514, 531, 487, 499, 505, 558, 511, 531, 521, 507, 479, 543, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[475, 510, 501, 493, 511, 509, 556, 511, 464, 489, 529, 510, 524, 489, 503, 508, 494, 510, 523, 524, 519, 546, 512, 475, 532, 488, 493, 512, 528, 517, 517, 503, 536, 513, 517, 536, 530, 514, 471, 520, 499, 479, 504, 517, 517, 511, 545, 547, 524, 511, 538, 539, 490, 492, 536, 493, 521, 527, 509, 549, 517, 511, 484, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 496, 515, 515, 508, 516, 503, 499, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 492, 523, 509, 528, 529, 539, 3, 1, 0, 0, 0, 0, 0, 0, 527, 502, 529, 507, 526, 493, 512, 504, 5, 2, 0, 0, 0, 0, 0, 0, 535, 525, 496, 558, 483, 517, 464, 490, 0, 0, 0, 0, 0, 0, 0, 0, 491, 547, 513, 528, 487, 527, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 469, 505, 533, 501, 499, 485, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 540, 529, 516, 549, 505, 500, 467, 538, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 547, 526, 498, 530, 517, 526, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4111, 4107, 4068, 4107, 4001, 4144, 4168] avg=4096.0000 max=4168 max_rank=7 min=4001 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 481, 511, 477, 542, 540, 486, 519, 0, 0, 0, 0, 0, 0, 0, 0, 519, 533, 505, 508, 497, 497, 525, 498, 3, 2, 0, 0, 0, 0, 0, 0, 491, 537, 522, 502, 509, 504, 496, 531, 5, 2, 0, 0, 0, 0, 0, 0, 522, 529, 515, 515, 489, 473, 510, 532, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 502, 514, 500, 538, 525, 551, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 544, 505, 512, 484, 538, 525, 0, 0, 0, 0, 0, 0, 0, 0, 498, 529, 469, 500, 482, 527, 507, 530, 0, 0, 0, 0, 0, 0, 0, 0, 517, 487, 489, 479, 525, 516, 522, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4087, 4099, 4085, 4170, 4147, 4042, 4063] avg=4096.0000 max=4170 max_rank=4 min=4042 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[540, 544, 472, 477, 512, 558, 546, 465, 0, 0, 0, 0, 0, 0, 0, 0, 526, 484, 535, 517, 493, 524, 512, 534, 3, 1, 0, 0, 0, 0, 0, 0, 512, 477, 483, 503, 481, 533, 498, 502, 5, 2, 0, 0, 0, 0, 0, 0, 494, 511, 518, 498, 470, 525, 576, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 489, 479, 500, 512, 520, 500, 484, 0, 0, 0, 0, 0, 0, 0, 0, 518, 496, 501, 514, 530, 552, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 526, 512, 475, 528, 541, 497, 0, 0, 0, 0, 0, 0, 0, 0, 505, 553, 515, 528, 502, 496, 528, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4129, 3996, 4127, 3990, 4150, 4123, 4139] avg=4096.0000 max=4150 max_rank=5 min=3990 min_rank=4 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[488, 519, 484, 501, 511, 495, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 522, 497, 507, 558, 513, 515, 528, 3, 1, 0, 0, 0, 0, 0, 0, 526, 492, 549, 522, 488, 463, 497, 508, 5, 2, 0, 0, 0, 0, 0, 0, 498, 488, 522, 526, 524, 513, 539, 500, 0, 0, 0, 0, 0, 0, 0, 0, 540, 519, 520, 513, 509, 508, 493, 518, 0, 0, 0, 0, 0, 0, 0, 0, 469, 504, 550, 505, 514, 525, 513, 481, 0, 0, 0, 0, 0, 0, 0, 0, 519, 525, 523, 524, 499, 495, 532, 526, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 511, 488, 526, 510, 511, 519, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4155, 4052, 4110, 4120, 4061, 4143, 4070] avg=4096.0000 max=4155 max_rank=1 min=4052 min_rank=2 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 508, 554, 493, 558, 551, 462, 501, 0, 0, 0, 0, 0, 0, 0, 0, 547, 492, 503, 549, 484, 504, 482, 534, 3, 2, 0, 0, 0, 0, 0, 0, 518, 493, 513, 511, 541, 511, 510, 531, 5, 2, 0, 0, 0, 0, 0, 0, 529, 490, 518, 502, 510, 481, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0, 489, 518, 468, 567, 484, 542, 531, 497, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 542, 521, 495, 502, 499, 534, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 511, 496, 483, 523, 486, 525, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 481, 498, 543, 502, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4100, 4135, 4089, 4096, 4113, 4050, 4036] avg=4096.0000 max=4149 max_rank=0 min=4036 min_rank=7 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[534, 506, 474, 516, 519, 493, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 494, 529, 509, 514, 506, 525, 3, 1, 0, 0, 0, 0, 0, 0, 518, 525, 488, 541, 521, 507, 482, 527, 5, 2, 0, 0, 0, 0, 0, 0, 508, 514, 522, 532, 480, 542, 477, 538, 0, 0, 0, 0, 0, 0, 0, 0, 516, 505, 486, 513, 531, 521, 526, 575, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 535, 514, 538, 511, 498, 510, 0, 0, 0, 0, 0, 0, 0, 0, 521, 489, 498, 520, 496, 519, 518, 517, 0, 0, 0, 0, 0, 0, 0, 0, 507, 506, 486, 478, 509, 522, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4110, 4116, 4113, 4173, 4132, 4078, 4023] avg=4096.0000 max=4173 max_rank=4 min=4023 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[552, 522, 520, 537, 531, 521, 520, 532, 0, 0, 0, 0, 0, 0, 0, 0, 497, 530, 522, 496, 516, 514, 476, 493, 3, 2, 0, 0, 0, 0, 0, 0, 506, 492, 537, 527, 532, 514, 540, 491, 5, 2, 0, 0, 0, 0, 0, 0, 498, 535, 545, 530, 497, 518, 459, 505, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 513, 494, 483, 482, 520, 460, 0, 0, 0, 0, 0, 0, 0, 0, 482, 513, 521, 502, 530, 507, 486, 506, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 515, 514, 531, 487, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 558, 511, 531, 521, 507, 479, 543, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4235, 4049, 4146, 4087, 3984, 4047, 4055, 4165] avg=4096.0000 max=4235 max_rank=0 min=3984 min_rank=4 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[475, 510, 501, 493, 498, 509, 556, 511, 0, 0, 0, 0, 0, 0, 0, 0, 464, 489, 529, 510, 524, 489, 503, 508, 5, 6, 0, 0, 0, 0, 0, 0, 494, 510, 523, 524, 519, 546, 512, 475, 10, 4, 0, 0, 0, 0, 0, 0, 532, 488, 493, 511, 528, 517, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 527, 513, 517, 536, 530, 514, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0, 499, 479, 502, 517, 517, 511, 545, 547, 0, 0, 0, 0, 0, 0, 0, 0, 524, 511, 538, 539, 490, 492, 536, 493, 7, 2, 1, 0, 0, 0, 0, 0, 521, 527, 509, 549, 517, 511, 484, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4027, 4117, 4089, 4128, 4117, 4133, 4104] avg=4096.0000 max=4133 max_rank=6 min=4027 min_rank=1 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4352, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 14:57:41] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 279.3 | throughput per GPU (TFLOP/s/GPU): 442.2 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.516894E+00 | loss scale: 1.0 | grad norm: 30.428 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[475, 547, 498, 505, 535, 461, 522, 494, 492, 500, 532, 482, 551, 514, 528, 533, 499, 488, 506, 503, 526, 516, 521, 481, 523, 540, 517, 543, 528, 509, 517, 524, 534, 546, 483, 498, 511, 495, 548, 517, 532, 540, 498, 452, 564, 529, 538, 523, 493, 513, 510, 491, 495, 528, 542, 464, 475, 515, 518, 513, 494, 516, 508, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[527, 526, 537, 496, 500, 516, 521, 519, 507, 535, 519, 507, 553, 510, 523, 523, 483, 485, 543, 524, 537, 512, 483, 454, 553, 543, 510, 540, 510, 486, 494, 517, 518, 499, 530, 474, 518, 530, 513, 488, 502, 505, 505, 540, 477, 495, 523, 492, 522, 520, 525, 502, 496, 491, 517, 481, 501, 525, 511, 487, 515, 519, 543, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[519, 508, 509, 517, 455, 519, 516, 515, 508, 517, 511, 480, 541, 568, 500, 474, 488, 467, 545, 522, 517, 497, 513, 499, 528, 526, 541, 531, 515, 519, 504, 547, 501, 483, 507, 522, 505, 560, 509, 492, 525, 529, 534, 531, 495, 496, 540, 493, 482, 505, 521, 513, 507, 494, 505, 518, 518, 493, 483, 497, 528, 541, 528, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[512, 489, 500, 488, 495, 521, 479, 534, 525, 516, 542, 538, 524, 484, 568, 512, 504, 546, 512, 498, 510, 464, 517, 536, 528, 532, 525, 539, 510, 494, 494, 503, 492, 514, 501, 515, 518, 491, 507, 550, 503, 556, 512, 495, 498, 466, 516, 535, 483, 480, 501, 558, 509, 539, 516, 554, 490, 504, 501, 500, 463, 489, 528, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[524, 512, 482, 524, 529, 515, 512, 506, 483, 494, 541, 560, 508, 519, 514, 509, 517, 502, 555, 569, 501, 523, 496, 480, 548, 509, 524, 499, 530, 480, 539, 491, 510, 507, 499, 505, 526, 517, 512, 554, 516, 546, 487, 547, 529, 547, 503, 494, 467, 485, 494, 527, 484, 493, 485, 496, 506, 502, 521, 489, 533, 456, 543, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[575, 513, 504, 502, 509, 517, 516, 510, 525, 501, 499, 533, 456, 485, 505, 536, 497, 522, 512, 541, 498, 505, 516, 559, 513, 492, 473, 497, 480, 498, 555, 534, 518, 504, 505, 500, 533, 529, 512, 518, 503, 507, 532, 513, 510, 509, 522, 498, 525, 549, 491, 523, 561, 477, 490, 522, 520, 486, 499, 509, 476, 529, 527, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[498, 544, 545, 487, 517, 510, 529, 525, 477, 465, 489, 512, 508, 498, 533, 514, 563, 529, 536, 512, 507, 531, 486, 481, 504, 496, 445, 526, 543, 545, 541, 528, 512, 519, 518, 502, 542, 555, 563, 514, 482, 485, 488, 493, 539, 519, 483, 543, 506, 518, 520, 489, 502, 517, 509, 514, 505, 447, 515, 486, 495, 516, 513, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[495, 536, 525, 508, 494, 495, 490, 504, 510, 523, 484, 468, 511, 507, 514, 512, 523, 505, 499, 503, 507, 520, 494, 522, 513, 518, 510, 504, 507, 553, 515, 554, 520, 493, 530, 540, 483, 518, 491, 535, 506, 550, 489, 539, 547, 485, 503, 550, 527, 537, 509, 484, 504, 500, 547, 508, 483, 487, 491, 506, 537, 513, 506, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[475, 547, 498, 505, 535, 461, 522, 494, 7, 0, 0, 0, 0, 0, 0, 0, 492, 500, 532, 482, 551, 514, 523, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 488, 506, 503, 526, 516, 521, 481, 5, 2, 0, 0, 0, 0, 0, 0, 476, 540, 517, 543, 528, 509, 517, 524, 0, 0, 0, 0, 0, 0, 0, 0, 534, 546, 483, 498, 511, 474, 548, 517, 0, 0, 0, 0, 0, 0, 0, 0, 532, 524, 498, 452, 564, 529, 538, 523, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 510, 491, 495, 528, 542, 464, 21, 7, 0, 0, 0, 0, 0, 0, 475, 515, 518, 513, 494, 516, 508, 475, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4127, 4047, 4154, 4111, 4160, 4064, 4061] avg=4096.0000 max=4160 max_rank=5 min=4044 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[527, 526, 537, 496, 500, 516, 521, 519, 7, 0, 0, 0, 0, 0, 0, 0, 507, 535, 519, 507, 553, 510, 518, 523, 0, 0, 0, 0, 0, 0, 0, 0, 483, 485, 543, 524, 537, 512, 483, 454, 5, 2, 0, 0, 0, 0, 0, 0, 503, 543, 510, 540, 510, 486, 494, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 499, 530, 474, 518, 508, 513, 488, 0, 0, 0, 0, 0, 0, 0, 0, 502, 490, 505, 540, 477, 495, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 525, 502, 496, 491, 517, 481, 22, 6, 0, 0, 0, 0, 0, 0, 501, 525, 511, 487, 515, 519, 543, 511, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4172, 4028, 4103, 4048, 4024, 4082, 4162] avg=4096.0000 max=4172 max_rank=1 min=4024 min_rank=5 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16316 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[519, 508, 509, 517, 455, 519, 516, 515, 7, 0, 0, 0, 0, 0, 0, 0, 508, 517, 511, 480, 541, 568, 495, 474, 0, 0, 0, 0, 0, 0, 0, 0, 488, 467, 545, 522, 517, 497, 513, 499, 5, 2, 0, 0, 0, 0, 0, 0, 480, 526, 541, 531, 515, 519, 504, 547, 0, 0, 0, 0, 0, 0, 0, 0, 501, 483, 507, 522, 505, 537, 509, 492, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 534, 531, 495, 496, 540, 493, 0, 0, 0, 0, 0, 0, 0, 0, 482, 505, 521, 513, 507, 494, 505, 518, 23, 6, 0, 0, 0, 0, 0, 0, 518, 493, 483, 497, 528, 541, 528, 497, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4094, 4055, 4163, 4056, 4128, 4074, 4133] avg=4096.0000 max=4163 max_rank=3 min=4055 min_rank=2 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[524, 512, 482, 524, 529, 515, 512, 506, 7, 0, 0, 0, 0, 0, 0, 0, 483, 494, 541, 560, 508, 519, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 555, 569, 501, 523, 496, 480, 5, 2, 0, 0, 0, 0, 0, 0, 498, 509, 524, 499, 530, 480, 539, 491, 0, 0, 0, 0, 0, 0, 0, 0, 510, 507, 499, 505, 526, 496, 512, 554, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 487, 547, 529, 547, 503, 494, 0, 0, 0, 0, 0, 0, 0, 0, 467, 485, 494, 527, 484, 493, 485, 496, 21, 7, 0, 0, 0, 0, 0, 0, 506, 502, 521, 489, 533, 456, 543, 493, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4123, 4150, 4070, 4109, 4153, 3959, 4093] avg=4096.0000 max=4153 max_rank=5 min=3959 min_rank=6 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[512, 489, 500, 488, 495, 521, 479, 534, 8, 0, 0, 0, 0, 0, 0, 0, 525, 516, 542, 538, 524, 484, 563, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 546, 512, 498, 510, 464, 517, 536, 5, 2, 0, 0, 0, 0, 0, 0, 480, 532, 525, 539, 510, 494, 494, 503, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 501, 515, 518, 471, 507, 550, 0, 0, 0, 0, 0, 0, 0, 0, 503, 539, 512, 495, 498, 466, 516, 535, 0, 0, 0, 0, 0, 0, 0, 0, 483, 480, 501, 558, 509, 539, 516, 554, 20, 7, 0, 0, 0, 0, 0, 0, 490, 504, 501, 500, 463, 489, 528, 545, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4204, 4094, 4077, 4068, 4064, 4167, 4068] avg=4096.0000 max=4204 max_rank=1 min=4026 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[495, 536, 525, 508, 494, 495, 490, 504, 7, 0, 0, 0, 0, 0, 0, 0, 510, 523, 484, 468, 511, 507, 509, 512, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 499, 503, 507, 520, 494, 522, 5, 2, 0, 0, 0, 0, 0, 0, 466, 518, 510, 504, 507, 553, 515, 554, 0, 0, 0, 0, 0, 0, 0, 0, 520, 493, 530, 540, 483, 497, 491, 535, 0, 0, 0, 0, 0, 0, 0, 0, 506, 534, 489, 539, 547, 485, 503, 550, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 509, 484, 504, 500, 547, 508, 21, 7, 0, 0, 0, 0, 0, 0, 483, 487, 491, 506, 537, 513, 506, 527, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4024, 4080, 4127, 4089, 4153, 4144, 4097] avg=4096.0000 max=4153 max_rank=5 min=4024 min_rank=1 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16483 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 544, 545, 487, 517, 510, 529, 525, 11, 0, 0, 0, 0, 0, 0, 0, 477, 465, 489, 512, 508, 498, 526, 514, 0, 0, 0, 0, 0, 0, 0, 0, 563, 529, 536, 512, 507, 531, 486, 481, 7, 5, 0, 0, 0, 0, 0, 0, 455, 496, 445, 526, 543, 545, 541, 528, 0, 0, 0, 0, 0, 0, 0, 0, 512, 519, 518, 502, 542, 520, 563, 514, 0, 0, 0, 0, 0, 0, 0, 0, 482, 460, 488, 493, 539, 519, 483, 543, 0, 0, 0, 0, 0, 0, 0, 0, 506, 518, 520, 489, 502, 517, 509, 514, 28, 9, 0, 0, 0, 0, 0, 0, 505, 447, 515, 486, 495, 516, 513, 535, 49, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 3989, 4157, 4079, 4190, 4007, 4112, 4068] avg=4096.0000 max=4190 max_rank=4 min=3989 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[575, 513, 504, 502, 509, 517, 516, 510, 7, 0, 0, 0, 0, 0, 0, 0, 525, 501, 499, 533, 456, 485, 500, 536, 0, 0, 0, 0, 0, 0, 0, 0, 497, 522, 512, 541, 498, 505, 516, 559, 5, 2, 0, 0, 0, 0, 0, 0, 466, 492, 473, 497, 480, 498, 555, 534, 0, 0, 0, 0, 0, 0, 0, 0, 518, 504, 505, 500, 533, 507, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 532, 513, 510, 509, 522, 498, 0, 0, 0, 0, 0, 0, 0, 0, 525, 549, 491, 523, 561, 477, 490, 522, 22, 6, 0, 0, 0, 0, 0, 0, 520, 486, 499, 509, 476, 529, 527, 493, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4035, 4157, 3995, 4097, 4079, 4166, 4086] avg=4096.0000 max=4166 max_rank=6 min=3995 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4096, 4096, 4096, 4096, 4224, 4096, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[527, 533, 509, 548, 514, 529, 507, 503, 475, 490, 524, 522, 504, 517, 511, 512, 527, 504, 483, 515, 507, 541, 530, 513, 520, 542, 491, 538, 484, 469, 526, 512, 491, 535, 492, 524, 477, 516, 499, 482, 515, 522, 505, 495, 522, 490, 524, 498, 521, 498, 539, 506, 501, 516, 554, 537, 495, 503, 493, 515, 567, 524, 499, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[540, 506, 487, 496, 503, 495, 482, 518, 507, 523, 490, 527, 476, 489, 497, 535, 517, 530, 526, 523, 528, 511, 504, 511, 533, 515, 512, 531, 521, 534, 520, 535, 509, 506, 517, 495, 524, 510, 531, 512, 507, 546, 502, 471, 529, 481, 519, 529, 495, 506, 514, 519, 520, 523, 496, 488, 525, 507, 526, 525, 508, 486, 492, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[508, 532, 493, 517, 522, 473, 523, 514, 515, 557, 521, 530, 505, 509, 481, 545, 528, 514, 500, 520, 529, 545, 551, 491, 481, 484, 508, 510, 492, 523, 478, 479, 495, 507, 493, 535, 510, 502, 519, 522, 514, 546, 468, 556, 533, 530, 518, 523, 530, 547, 447, 513, 512, 505, 525, 482, 466, 508, 509, 535, 500, 480, 526, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[547, 495, 516, 492, 503, 528, 502, 507, 500, 560, 533, 539, 487, 477, 526, 529, 527, 493, 506, 506, 513, 538, 502, 525, 465, 496, 552, 492, 509, 494, 492, 502, 509, 487, 513, 548, 535, 490, 523, 508, 534, 479, 478, 489, 496, 510, 543, 519, 480, 531, 488, 534, 517, 483, 530, 530, 507, 486, 533, 500, 540, 534, 523, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[528, 548, 520, 505, 518, 493, 558, 497, 510, 496, 497, 511, 507, 472, 521, 520, 517, 520, 509, 501, 506, 529, 506, 531, 524, 523, 537, 519, 482, 539, 509, 495, 542, 522, 512, 552, 498, 481, 507, 473, 523, 533, 526, 536, 465, 526, 510, 508, 521, 481, 518, 496, 527, 486, 531, 535, 490, 484, 533, 534, 496, 455, 535, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[514, 501, 526, 535, 506, 539, 479, 559, 484, 508, 535, 507, 496, 539, 515, 484, 482, 515, 496, 547, 523, 525, 494, 502, 539, 544, 476, 506, 506, 510, 547, 521, 503, 532, 528, 515, 490, 537, 515, 497, 530, 535, 552, 484, 487, 525, 500, 478, 515, 510, 527, 511, 503, 501, 510, 485, 514, 543, 508, 521, 480, 477, 475, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[516, 528, 537, 530, 515, 535, 537, 504, 528, 517, 470, 533, 509, 531, 502, 503, 507, 488, 532, 509, 529, 500, 499, 539, 489, 529, 526, 539, 527, 517, 498, 498, 528, 489, 537, 512, 482, 486, 541, 512, 509, 514, 506, 529, 469, 445, 504, 535, 509, 508, 502, 521, 512, 486, 517, 502, 511, 514, 476, 539, 502, 514, 525, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[553, 505, 554, 498, 493, 539, 530, 506, 512, 536, 513, 488, 491, 499, 528, 508, 482, 546, 545, 497, 544, 483, 549, 516, 515, 488, 490, 544, 514, 507, 495, 530, 459, 479, 524, 519, 505, 524, 515, 518, 545, 486, 537, 531, 518, 505, 509, 507, 545, 521, 503, 489, 501, 533, 507, 528, 485, 473, 480, 514, 531, 474, 493, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 533, 509, 548, 514, 529, 507, 503, 0, 0, 0, 0, 0, 0, 0, 0, 475, 490, 524, 522, 504, 517, 511, 512, 10, 0, 0, 0, 0, 0, 0, 0, 527, 504, 483, 515, 478, 541, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 520, 542, 491, 538, 484, 469, 526, 512, 5, 0, 0, 0, 0, 0, 0, 0, 491, 535, 492, 524, 477, 516, 499, 482, 9, 4, 0, 0, 0, 0, 0, 0, 515, 522, 505, 495, 522, 490, 524, 498, 1, 0, 0, 0, 0, 0, 0, 0, 521, 498, 539, 506, 501, 516, 554, 537, 4, 0, 0, 0, 0, 0, 0, 0, 495, 503, 493, 515, 567, 524, 499, 486, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4065, 4091, 4087, 4029, 4072, 4176, 4115] avg=4096.0000 max=4176 max_rank=6 min=4029 min_rank=4 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[501, 506, 487, 496, 503, 495, 482, 518, 0, 0, 0, 0, 0, 0, 0, 0, 507, 523, 490, 527, 476, 489, 497, 535, 10, 0, 0, 0, 0, 0, 0, 0, 517, 530, 526, 523, 498, 511, 504, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 515, 512, 531, 521, 534, 520, 535, 5, 0, 0, 0, 0, 0, 0, 0, 509, 506, 517, 495, 524, 510, 531, 512, 9, 5, 0, 0, 0, 0, 0, 0, 507, 546, 502, 471, 529, 481, 519, 529, 1, 0, 0, 0, 0, 0, 0, 0, 495, 506, 514, 519, 520, 523, 496, 488, 5, 0, 0, 0, 0, 0, 0, 0, 525, 507, 526, 525, 508, 486, 492, 528, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 4054, 4120, 4206, 4118, 4085, 4066, 4131] avg=4096.0000 max=4206 max_rank=3 min=3988 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 495, 516, 492, 503, 528, 502, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 560, 533, 539, 487, 477, 526, 529, 10, 0, 0, 0, 0, 0, 0, 0, 527, 493, 506, 506, 484, 538, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 465, 496, 552, 492, 509, 494, 492, 502, 5, 0, 0, 0, 0, 0, 0, 0, 509, 487, 513, 548, 535, 490, 523, 508, 9, 5, 0, 0, 0, 0, 0, 0, 534, 479, 478, 489, 496, 510, 543, 519, 1, 0, 0, 0, 0, 0, 0, 0, 480, 531, 488, 534, 517, 483, 530, 530, 4, 0, 0, 0, 0, 0, 0, 0, 507, 486, 533, 500, 540, 534, 523, 538, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4161, 4081, 4007, 4127, 4049, 4097, 4195] avg=4096.0000 max=4195 max_rank=7 min=4007 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[472, 532, 493, 517, 522, 473, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 515, 557, 521, 530, 505, 509, 481, 545, 10, 0, 0, 0, 0, 0, 0, 0, 528, 514, 500, 520, 499, 545, 551, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 484, 508, 510, 492, 523, 478, 479, 5, 0, 0, 0, 0, 0, 0, 0, 495, 507, 493, 535, 510, 502, 519, 522, 9, 4, 0, 0, 0, 0, 0, 0, 514, 546, 468, 556, 533, 530, 518, 523, 1, 0, 0, 0, 0, 0, 0, 0, 530, 547, 447, 513, 512, 505, 525, 482, 5, 0, 0, 0, 0, 0, 0, 0, 466, 508, 509, 535, 500, 480, 526, 534, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4173, 4148, 3960, 4096, 4189, 4066, 4090] avg=4096.0000 max=4189 max_rank=5 min=3960 min_rank=3 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[478, 501, 526, 535, 506, 539, 479, 559, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 535, 507, 496, 539, 515, 484, 10, 0, 0, 0, 0, 0, 0, 0, 482, 515, 496, 547, 493, 525, 494, 502, 0, 0, 0, 0, 0, 0, 0, 0, 539, 544, 476, 506, 506, 510, 547, 521, 5, 0, 0, 0, 0, 0, 0, 0, 503, 532, 528, 515, 490, 537, 515, 497, 9, 4, 0, 0, 0, 0, 0, 0, 530, 535, 552, 484, 487, 525, 500, 478, 1, 0, 0, 0, 0, 0, 0, 0, 515, 510, 527, 511, 503, 501, 510, 485, 5, 0, 0, 0, 0, 0, 0, 0, 514, 543, 508, 521, 480, 477, 475, 520, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4078, 4054, 4154, 4130, 4092, 4067, 4070] avg=4096.0000 max=4154 max_rank=3 min=4054 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 528, 537, 530, 515, 535, 537, 504, 0, 0, 0, 0, 0, 0, 0, 0, 528, 517, 470, 533, 509, 531, 502, 503, 10, 0, 0, 0, 0, 0, 0, 0, 507, 488, 532, 509, 499, 500, 499, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 526, 539, 527, 517, 498, 498, 5, 0, 0, 0, 0, 0, 0, 0, 528, 489, 537, 512, 482, 486, 541, 512, 9, 4, 0, 0, 0, 0, 0, 0, 509, 514, 506, 529, 469, 445, 504, 535, 1, 0, 0, 0, 0, 0, 0, 0, 509, 508, 502, 521, 512, 486, 517, 502, 5, 0, 0, 0, 0, 0, 0, 0, 511, 514, 476, 539, 502, 514, 525, 511, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4103, 4073, 4128, 4100, 4012, 4062, 4124] avg=4096.0000 max=4166 max_rank=0 min=4012 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[483, 548, 520, 505, 518, 493, 558, 497, 0, 0, 0, 0, 0, 0, 0, 0, 510, 496, 497, 511, 507, 472, 521, 520, 14, 0, 0, 0, 0, 0, 0, 0, 517, 520, 509, 501, 458, 529, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 537, 519, 482, 539, 509, 495, 10, 0, 0, 0, 0, 0, 0, 0, 542, 522, 512, 552, 498, 481, 507, 473, 14, 9, 0, 0, 0, 0, 0, 0, 523, 533, 526, 536, 465, 526, 510, 508, 3, 0, 0, 0, 0, 0, 0, 0, 521, 481, 518, 496, 527, 486, 531, 535, 7, 0, 0, 0, 0, 0, 0, 0, 490, 484, 533, 534, 496, 455, 535, 484, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4048, 4071, 4138, 4110, 4130, 4102, 4047] avg=4096.0000 max=4138 max_rank=3 min=4047 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 505, 554, 498, 493, 539, 530, 506, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 513, 488, 491, 499, 528, 508, 11, 0, 0, 0, 0, 0, 0, 0, 482, 546, 545, 497, 512, 483, 549, 516, 0, 0, 0, 0, 0, 0, 0, 0, 515, 488, 490, 544, 514, 507, 495, 530, 5, 0, 0, 0, 0, 0, 0, 0, 459, 479, 524, 519, 505, 524, 515, 518, 10, 5, 0, 0, 0, 0, 0, 0, 545, 486, 537, 531, 518, 505, 509, 507, 1, 0, 0, 0, 0, 0, 0, 0, 545, 521, 503, 489, 501, 533, 507, 528, 5, 0, 0, 0, 0, 0, 0, 0, 485, 473, 480, 514, 531, 474, 493, 512, 34, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4086, 4130, 4088, 4058, 4139, 4132, 3996] avg=4096.0000 max=4139 max_rank=0 min=3996 min_rank=7 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[539, 524, 482, 521, 540, 522, 476, 502, 514, 506, 507, 520, 577, 520, 505, 504, 480, 523, 535, 495, 520, 500, 538, 497, 523, 534, 476, 537, 487, 493, 499, 486, 536, 501, 546, 466, 531, 525, 550, 503, 484, 483, 510, 525, 556, 513, 491, 495, 512, 464, 509, 511, 511, 468, 513, 518, 557, 500, 518, 519, 524, 508, 547, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[518, 542, 522, 565, 515, 534, 539, 532, 442, 530, 532, 545, 529, 495, 542, 545, 515, 542, 545, 488, 522, 505, 519, 508, 520, 529, 504, 501, 510, 492, 512, 510, 501, 512, 537, 497, 516, 535, 531, 518, 494, 505, 472, 521, 509, 521, 498, 427, 512, 487, 482, 502, 491, 519, 536, 522, 514, 514, 480, 515, 540, 487, 475, 449] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[509, 495, 491, 498, 487, 512, 533, 489, 518, 509, 539, 503, 493, 482, 516, 538, 528, 568, 531, 529, 492, 548, 471, 488, 501, 514, 481, 519, 562, 531, 486, 508, 551, 551, 475, 492, 525, 512, 484, 481, 502, 489, 512, 514, 528, 502, 521, 535, 515, 509, 544, 519, 497, 525, 518, 537, 501, 524, 516, 523, 492, 517, 471, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[538, 530, 525, 532, 489, 536, 478, 537, 543, 494, 509, 511, 485, 496, 495, 507, 518, 479, 527, 481, 520, 534, 531, 496, 505, 517, 519, 494, 496, 521, 500, 556, 497, 499, 507, 524, 494, 501, 526, 508, 495, 544, 496, 501, 544, 507, 463, 541, 495, 524, 482, 531, 514, 516, 524, 509, 505, 538, 513, 509, 508, 537, 509, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[523, 529, 531, 533, 527, 518, 494, 528, 548, 552, 535, 527, 539, 517, 501, 521, 505, 515, 510, 550, 521, 513, 532, 557, 531, 498, 519, 479, 478, 493, 505, 541, 533, 498, 509, 504, 513, 485, 487, 535, 492, 488, 514, 514, 544, 476, 492, 531, 536, 524, 494, 483, 478, 505, 502, 529, 476, 475, 473, 491, 468, 504, 522, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[535, 517, 507, 527, 490, 512, 506, 517, 506, 505, 527, 501, 523, 514, 508, 535, 496, 505, 484, 540, 507, 506, 543, 498, 512, 497, 515, 529, 521, 509, 493, 544, 511, 498, 466, 494, 477, 512, 511, 487, 502, 542, 534, 500, 494, 519, 526, 534, 508, 501, 517, 503, 538, 497, 512, 477, 527, 502, 540, 546, 509, 521, 528, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 519, 527, 497, 496, 532, 506, 532, 494, 492, 543, 539, 525, 486, 481, 516, 529, 507, 512, 522, 497, 546, 504, 533, 517, 543, 500, 517, 514, 513, 508, 531, 518, 504, 490, 535, 517, 497, 506, 513, 532, 527, 525, 535, 510, 533, 509, 528, 502, 483, 519, 495, 482, 495, 499, 480, 536, 500, 529, 535, 480, 506, 475, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[495, 507, 537, 525, 540, 485, 487, 490, 523, 513, 530, 484, 498, 465, 522, 524, 531, 500, 537, 479, 516, 511, 491, 491, 543, 507, 501, 489, 513, 554, 508, 498, 527, 537, 500, 495, 500, 539, 477, 568, 497, 496, 532, 540, 484, 480, 501, 495, 532, 547, 506, 506, 529, 500, 529, 522, 540, 527, 519, 513, 476, 531, 523, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[539, 524, 482, 490, 540, 522, 476, 502, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 486, 520, 577, 520, 505, 504, 0, 0, 0, 0, 0, 0, 0, 0, 480, 523, 512, 495, 520, 500, 538, 497, 0, 0, 0, 0, 0, 0, 0, 0, 523, 534, 476, 537, 487, 493, 499, 486, 3, 0, 0, 0, 0, 0, 0, 0, 536, 501, 546, 466, 531, 525, 550, 503, 11, 0, 0, 0, 0, 0, 0, 0, 484, 483, 510, 525, 556, 513, 491, 495, 10, 7, 0, 0, 0, 0, 0, 0, 512, 464, 509, 511, 511, 468, 513, 518, 27, 0, 0, 0, 0, 0, 0, 0, 557, 500, 518, 519, 524, 508, 547, 492, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4132, 4065, 4038, 4169, 4074, 4033, 4182] avg=4096.0000 max=4182 max_rank=7 min=4033 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[518, 542, 522, 531, 515, 534, 539, 532, 0, 0, 0, 0, 0, 0, 0, 0, 442, 530, 511, 545, 529, 495, 542, 545, 0, 0, 0, 0, 0, 0, 0, 0, 515, 542, 520, 488, 522, 505, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 529, 504, 501, 510, 492, 512, 510, 3, 0, 0, 0, 0, 0, 0, 0, 501, 512, 537, 497, 516, 535, 531, 518, 11, 0, 0, 0, 0, 0, 0, 0, 494, 505, 472, 521, 509, 521, 498, 427, 11, 7, 0, 0, 0, 0, 0, 0, 512, 487, 482, 502, 491, 519, 536, 522, 29, 0, 0, 0, 0, 0, 0, 0, 514, 514, 480, 515, 540, 487, 475, 449, 14, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4233, 4139, 4119, 4081, 4158, 3965, 4080, 3993] avg=4096.0000 max=4233 max_rank=0 min=3965 min_rank=5 imbalance_ratio=1.033447 max_load_reduction=28535 max_load_reduction_pct=0.870819 moved_to_echo=16196 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 495, 491, 468, 487, 512, 533, 489, 0, 0, 0, 0, 0, 0, 0, 0, 518, 509, 518, 503, 493, 482, 516, 538, 0, 0, 0, 0, 0, 0, 0, 0, 528, 568, 508, 529, 492, 548, 471, 488, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 481, 519, 562, 531, 486, 508, 3, 0, 0, 0, 0, 0, 0, 0, 551, 551, 475, 492, 525, 512, 484, 481, 11, 0, 0, 0, 0, 0, 0, 0, 502, 489, 512, 514, 528, 502, 521, 535, 10, 7, 0, 0, 0, 0, 0, 0, 515, 509, 544, 519, 497, 525, 518, 537, 26, 0, 0, 0, 0, 0, 0, 0, 501, 524, 516, 523, 492, 517, 471, 517, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3984, 4077, 4132, 4105, 4082, 4120, 4190, 4078] avg=4096.0000 max=4190 max_rank=6 min=3984 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 519, 527, 467, 496, 532, 506, 532, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 522, 539, 525, 486, 481, 516, 0, 0, 0, 0, 0, 0, 0, 0, 529, 507, 489, 522, 497, 546, 504, 533, 0, 0, 0, 0, 0, 0, 0, 0, 517, 543, 500, 517, 514, 513, 508, 531, 3, 0, 0, 0, 0, 0, 0, 0, 518, 504, 490, 535, 517, 497, 506, 513, 11, 0, 0, 0, 0, 0, 0, 0, 532, 527, 525, 535, 510, 533, 509, 528, 10, 7, 0, 0, 0, 0, 0, 0, 502, 483, 519, 495, 482, 495, 499, 480, 26, 0, 0, 0, 0, 0, 0, 0, 536, 500, 529, 535, 480, 506, 475, 486, 13, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4055, 4127, 4146, 4091, 4216, 3981, 4064] avg=4096.0000 max=4216 max_rank=5 min=3981 min_rank=6 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[538, 530, 525, 499, 489, 536, 478, 537, 0, 0, 0, 0, 0, 0, 0, 0, 543, 494, 488, 511, 485, 496, 495, 507, 0, 0, 0, 0, 0, 0, 0, 0, 518, 479, 504, 481, 520, 534, 531, 496, 0, 0, 0, 0, 0, 0, 0, 0, 505, 517, 519, 494, 496, 521, 500, 556, 3, 0, 0, 0, 0, 0, 0, 0, 497, 499, 507, 524, 494, 501, 526, 508, 11, 0, 0, 0, 0, 0, 0, 0, 495, 544, 496, 501, 544, 507, 463, 541, 10, 7, 0, 0, 0, 0, 0, 0, 495, 524, 482, 531, 514, 516, 524, 509, 28, 0, 0, 0, 0, 0, 0, 0, 505, 538, 513, 509, 508, 537, 509, 508, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4019, 4063, 4111, 4067, 4108, 4123, 4145] avg=4096.0000 max=4145 max_rank=7 min=4019 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[495, 507, 537, 493, 540, 485, 487, 490, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 509, 484, 498, 465, 522, 524, 0, 0, 0, 0, 0, 0, 0, 0, 531, 500, 514, 479, 516, 511, 491, 491, 0, 0, 0, 0, 0, 0, 0, 0, 543, 507, 501, 489, 513, 554, 508, 498, 3, 0, 0, 0, 0, 0, 0, 0, 527, 537, 500, 495, 500, 539, 477, 568, 11, 0, 0, 0, 0, 0, 0, 0, 497, 496, 532, 540, 484, 480, 501, 495, 10, 7, 0, 0, 0, 0, 0, 0, 532, 547, 506, 506, 529, 500, 529, 522, 27, 0, 0, 0, 0, 0, 0, 0, 540, 527, 519, 513, 476, 531, 523, 506, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4038, 4033, 4116, 4154, 4042, 4198, 4153] avg=4096.0000 max=4198 max_rank=6 min=4033 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16547 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 529, 531, 500, 527, 518, 494, 528, 0, 0, 0, 0, 0, 0, 0, 0, 548, 552, 514, 527, 539, 517, 501, 521, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 487, 550, 521, 513, 532, 557, 0, 0, 0, 0, 0, 0, 0, 0, 531, 498, 519, 479, 478, 493, 505, 541, 3, 0, 0, 0, 0, 0, 0, 0, 533, 498, 509, 504, 513, 485, 487, 535, 11, 0, 0, 0, 0, 0, 0, 0, 492, 488, 514, 514, 544, 476, 492, 531, 10, 7, 0, 0, 0, 0, 0, 0, 536, 524, 494, 483, 478, 505, 502, 529, 28, 0, 0, 0, 0, 0, 0, 0, 476, 475, 473, 491, 468, 504, 522, 523, 13, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4219, 4180, 4047, 4075, 4068, 4079, 3950] avg=4096.0000 max=4219 max_rank=1 min=3950 min_rank=7 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16172 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[535, 517, 507, 488, 490, 512, 506, 517, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 497, 501, 523, 514, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 496, 505, 454, 540, 507, 506, 543, 498, 0, 0, 0, 0, 0, 0, 0, 0, 512, 497, 515, 529, 521, 509, 493, 544, 4, 0, 0, 0, 0, 0, 0, 0, 511, 498, 466, 494, 477, 512, 511, 487, 16, 0, 0, 0, 0, 0, 0, 0, 502, 542, 534, 500, 494, 519, 526, 534, 14, 10, 0, 0, 0, 0, 0, 0, 508, 501, 517, 503, 538, 497, 512, 477, 31, 0, 0, 0, 0, 0, 0, 0, 527, 502, 540, 546, 509, 521, 528, 506, 16, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4089, 4049, 4124, 3972, 4175, 4084, 4203] avg=4096.0000 max=4203 max_rank=7 min=3972 min_rank=4 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[519, 533, 542, 515, 526, 539, 506, 539, 488, 534, 509, 486, 553, 568, 538, 511, 495, 478, 500, 486, 471, 534, 466, 535, 497, 516, 498, 493, 550, 524, 505, 507, 505, 479, 497, 499, 528, 503, 522, 525, 544, 483, 508, 537, 519, 497, 480, 569, 475, 531, 529, 504, 487, 458, 518, 489, 523, 496, 476, 529, 480, 550, 526, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[487, 504, 502, 522, 514, 519, 507, 515, 524, 494, 501, 503, 511, 511, 486, 489, 546, 473, 464, 516, 515, 525, 497, 511, 511, 501, 518, 513, 504, 545, 528, 499, 524, 519, 506, 542, 506, 514, 525, 523, 526, 514, 549, 562, 529, 494, 464, 548, 515, 501, 527, 494, 533, 535, 488, 531, 521, 495, 523, 473, 510, 516, 512, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[513, 534, 485, 541, 503, 473, 536, 518, 497, 517, 524, 508, 513, 503, 510, 501, 503, 492, 524, 502, 529, 544, 527, 524, 479, 524, 488, 502, 458, 514, 481, 483, 525, 511, 520, 523, 502, 546, 499, 509, 557, 483, 529, 509, 558, 475, 530, 466, 517, 542, 524, 530, 467, 536, 530, 483, 510, 527, 518, 535, 509, 510, 502, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[537, 530, 529, 536, 539, 525, 520, 501, 506, 496, 527, 509, 502, 528, 513, 519, 505, 544, 527, 532, 498, 489, 533, 502, 485, 499, 521, 484, 469, 533, 528, 556, 531, 502, 551, 512, 542, 520, 480, 532, 515, 505, 509, 497, 469, 512, 505, 518, 520, 487, 481, 521, 483, 461, 506, 561, 515, 522, 502, 477, 497, 470, 538, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[508, 552, 561, 512, 517, 504, 496, 491, 550, 526, 514, 513, 472, 496, 519, 552, 541, 476, 469, 524, 555, 476, 510, 490, 530, 482, 511, 551, 490, 471, 499, 528, 477, 492, 519, 502, 532, 510, 503, 537, 507, 532, 516, 519, 523, 531, 458, 502, 533, 524, 509, 517, 533, 488, 529, 491, 529, 488, 509, 491, 508, 517, 548, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[521, 525, 480, 499, 500, 488, 519, 489, 542, 485, 498, 514, 544, 511, 502, 544, 526, 504, 486, 502, 496, 505, 501, 550, 500, 521, 546, 519, 520, 495, 515, 473, 517, 503, 544, 533, 515, 548, 518, 520, 529, 474, 531, 473, 482, 507, 509, 516, 505, 501, 530, 494, 512, 494, 533, 506, 524, 505, 484, 545, 525, 533, 523, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[499, 528, 539, 503, 495, 524, 514, 530, 503, 526, 519, 513, 539, 511, 499, 514, 459, 485, 552, 505, 563, 525, 508, 517, 502, 522, 564, 486, 502, 514, 548, 485, 504, 518, 522, 496, 520, 516, 518, 548, 515, 460, 542, 537, 531, 494, 527, 496, 497, 501, 534, 510, 498, 513, 488, 504, 489, 501, 499, 490, 496, 505, 519, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[557, 506, 497, 519, 519, 514, 549, 490, 511, 482, 539, 557, 544, 489, 510, 508, 503, 504, 496, 543, 509, 525, 487, 486, 512, 537, 498, 526, 486, 488, 482, 492, 504, 521, 515, 552, 461, 488, 495, 486, 516, 512, 510, 510, 472, 494, 523, 553, 521, 541, 512, 506, 508, 529, 545, 495, 549, 511, 493, 492, 526, 511, 539, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 498, 542, 515, 526, 539, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 488, 534, 509, 486, 534, 568, 538, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 478, 500, 486, 471, 534, 466, 535, 22, 3, 0, 0, 0, 0, 0, 0, 497, 516, 498, 493, 550, 524, 505, 507, 32, 0, 0, 0, 0, 0, 0, 0, 505, 479, 497, 499, 528, 503, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 544, 483, 508, 537, 519, 497, 480, 569, 0, 0, 0, 0, 0, 0, 0, 0, 475, 531, 529, 504, 487, 458, 518, 489, 12, 1, 0, 0, 0, 0, 0, 0, 523, 496, 476, 529, 480, 550, 526, 541, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4184, 4168, 3990, 4122, 4035, 4137, 4004, 4128] avg=4096.0000 max=4184 max_rank=0 min=3990 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[513, 499, 485, 541, 503, 473, 536, 518, 0, 0, 0, 0, 0, 0, 0, 0, 497, 517, 524, 508, 495, 503, 510, 501, 0, 0, 0, 0, 0, 0, 0, 0, 503, 492, 524, 502, 529, 544, 527, 524, 21, 3, 0, 0, 0, 0, 0, 0, 479, 524, 488, 502, 458, 514, 481, 483, 32, 0, 0, 0, 0, 0, 0, 0, 525, 511, 520, 523, 502, 546, 499, 487, 0, 0, 0, 0, 0, 0, 0, 0, 557, 483, 529, 509, 558, 475, 530, 466, 0, 0, 0, 0, 0, 0, 0, 0, 517, 542, 524, 530, 467, 536, 530, 483, 11, 1, 0, 0, 0, 0, 0, 0, 510, 527, 518, 535, 509, 510, 502, 536, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4055, 4169, 3961, 4113, 4107, 4141, 4154] avg=4096.0000 max=4169 max_rank=2 min=3961 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16515 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[508, 515, 561, 512, 517, 504, 496, 491, 0, 0, 0, 0, 0, 0, 0, 0, 550, 526, 514, 513, 456, 496, 519, 552, 0, 0, 0, 0, 0, 0, 0, 0, 541, 476, 469, 524, 555, 476, 510, 490, 22, 3, 0, 0, 0, 0, 0, 0, 530, 482, 511, 551, 490, 471, 499, 528, 34, 0, 0, 0, 0, 0, 0, 0, 477, 492, 519, 502, 532, 510, 503, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 532, 516, 519, 523, 531, 458, 502, 0, 0, 0, 0, 0, 0, 0, 0, 533, 524, 509, 517, 533, 488, 529, 491, 10, 1, 0, 0, 0, 0, 0, 0, 529, 488, 509, 491, 508, 517, 548, 508, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4126, 4066, 4096, 4049, 4088, 4135, 4104] avg=4096.0000 max=4135 max_rank=6 min=4049 min_rank=4 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 495, 529, 536, 539, 525, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 527, 509, 485, 528, 513, 519, 0, 0, 0, 0, 0, 0, 0, 0, 505, 544, 527, 532, 498, 489, 533, 502, 22, 3, 0, 0, 0, 0, 0, 0, 485, 499, 521, 484, 469, 533, 528, 556, 32, 0, 0, 0, 0, 0, 0, 0, 531, 502, 551, 512, 542, 520, 480, 509, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 509, 497, 469, 512, 505, 518, 0, 0, 0, 0, 0, 0, 0, 0, 520, 487, 481, 521, 483, 461, 506, 561, 11, 1, 0, 0, 0, 0, 0, 0, 515, 522, 502, 477, 497, 470, 538, 505, 6, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4182, 4083, 4155, 4107, 4147, 4030, 4032, 4032] avg=4096.0000 max=4182 max_rank=0 min=4030 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[487, 470, 502, 522, 514, 519, 507, 515, 0, 0, 0, 0, 0, 0, 0, 0, 524, 494, 501, 503, 493, 511, 486, 489, 0, 0, 0, 0, 0, 0, 0, 0, 546, 473, 464, 516, 515, 525, 497, 511, 22, 3, 0, 0, 0, 0, 0, 0, 511, 501, 518, 513, 504, 545, 528, 499, 31, 0, 0, 0, 0, 0, 0, 0, 524, 519, 506, 542, 506, 514, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 549, 562, 529, 494, 464, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 527, 494, 533, 535, 488, 531, 11, 1, 0, 0, 0, 0, 0, 0, 521, 495, 523, 473, 510, 516, 512, 494, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4001, 4072, 4150, 4136, 4186, 4136, 4051] avg=4096.0000 max=4186 max_rank=5 min=4001 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 490, 480, 499, 500, 488, 519, 489, 0, 0, 0, 0, 0, 0, 0, 0, 542, 485, 498, 514, 526, 511, 502, 544, 0, 0, 0, 0, 0, 0, 0, 0, 526, 504, 486, 502, 496, 505, 501, 550, 22, 3, 0, 0, 0, 0, 0, 0, 500, 521, 546, 519, 520, 495, 515, 473, 32, 0, 0, 0, 0, 0, 0, 0, 517, 503, 544, 533, 515, 548, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 529, 474, 531, 473, 482, 507, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 530, 494, 512, 494, 533, 506, 11, 1, 0, 0, 0, 0, 0, 0, 524, 505, 484, 545, 525, 533, 523, 515, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3986, 4122, 4095, 4121, 4175, 4021, 4087, 4161] avg=4096.0000 max=4175 max_rank=4 min=3986 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 487, 539, 503, 495, 524, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 519, 513, 506, 511, 499, 514, 0, 0, 0, 0, 0, 0, 0, 0, 459, 485, 552, 505, 563, 525, 508, 517, 26, 5, 0, 0, 0, 0, 0, 0, 502, 522, 564, 486, 502, 514, 548, 485, 36, 0, 0, 0, 0, 0, 0, 0, 504, 518, 522, 496, 520, 516, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 460, 542, 537, 531, 494, 527, 496, 7, 0, 0, 0, 0, 0, 0, 0, 497, 501, 534, 510, 498, 513, 488, 504, 15, 4, 0, 0, 0, 0, 0, 0, 489, 501, 499, 490, 496, 505, 519, 487, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4091, 4145, 4159, 4112, 4109, 4064, 3997] avg=4096.0000 max=4159 max_rank=3 min=3997 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[557, 472, 497, 519, 519, 514, 549, 490, 0, 0, 0, 0, 0, 0, 0, 0, 511, 482, 539, 557, 526, 489, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 503, 504, 496, 543, 509, 525, 487, 486, 20, 3, 0, 0, 0, 0, 0, 0, 512, 537, 498, 526, 486, 488, 482, 492, 31, 0, 0, 0, 0, 0, 0, 0, 504, 521, 515, 552, 461, 488, 495, 465, 0, 0, 0, 0, 0, 0, 0, 0, 516, 512, 510, 510, 472, 494, 523, 553, 0, 0, 0, 0, 0, 0, 0, 0, 521, 541, 512, 506, 508, 529, 545, 495, 11, 1, 0, 0, 0, 0, 0, 0, 549, 511, 493, 492, 526, 511, 539, 513, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4122, 4076, 4052, 4001, 4090, 4169, 4141] avg=4096.0000 max=4169 max_rank=6 min=4001 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[548, 531, 549, 554, 498, 529, 507, 545, 497, 556, 511, 515, 515, 525, 487, 513, 503, 536, 525, 545, 496, 490, 506, 510, 541, 519, 510, 475, 495, 491, 507, 509, 555, 545, 513, 526, 480, 500, 469, 507, 509, 481, 524, 480, 479, 488, 523, 472, 522, 541, 521, 512, 521, 510, 504, 518, 481, 486, 506, 500, 491, 537, 530, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[478, 509, 539, 557, 522, 514, 482, 538, 519, 500, 537, 502, 512, 487, 515, 505, 488, 479, 508, 553, 508, 525, 509, 516, 480, 550, 516, 512, 544, 489, 563, 510, 554, 512, 525, 490, 525, 503, 543, 525, 495, 526, 521, 512, 490, 520, 510, 492, 506, 526, 528, 507, 519, 514, 491, 514, 523, 502, 454, 488, 497, 478, 522, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[529, 511, 519, 567, 522, 525, 497, 508, 509, 538, 469, 518, 533, 519, 521, 527, 473, 533, 521, 459, 501, 534, 517, 449, 501, 516, 533, 473, 528, 535, 494, 503, 533, 463, 550, 524, 506, 494, 490, 509, 489, 541, 502, 507, 501, 522, 493, 493, 497, 537, 555, 517, 519, 495, 521, 486, 492, 484, 543, 519, 520, 521, 510, 553] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[492, 481, 527, 500, 508, 537, 501, 498, 497, 553, 461, 470, 511, 536, 533, 493, 530, 543, 501, 521, 518, 525, 503, 489, 521, 534, 517, 526, 546, 503, 507, 556, 513, 519, 511, 515, 514, 519, 514, 523, 507, 518, 476, 527, 495, 497, 568, 532, 493, 512, 461, 514, 514, 495, 478, 510, 501, 563, 539, 516, 504, 478, 469, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[485, 486, 530, 540, 514, 477, 496, 488, 504, 546, 520, 514, 519, 493, 505, 511, 535, 514, 565, 496, 496, 502, 506, 505, 521, 491, 523, 493, 525, 541, 489, 517, 508, 546, 523, 508, 489, 538, 534, 515, 517, 528, 519, 511, 539, 514, 554, 516, 498, 516, 491, 508, 491, 496, 482, 537, 516, 498, 513, 488, 519, 506, 517, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[541, 524, 490, 532, 509, 533, 496, 475, 513, 524, 530, 508, 473, 556, 553, 532, 503, 489, 509, 511, 525, 541, 487, 522, 500, 499, 529, 515, 494, 501, 528, 531, 504, 535, 517, 480, 532, 493, 532, 473, 524, 503, 501, 525, 524, 488, 506, 510, 497, 485, 504, 494, 508, 468, 518, 499, 507, 551, 533, 482, 509, 518, 554, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[512, 515, 542, 547, 522, 515, 544, 525, 512, 484, 529, 531, 478, 514, 542, 504, 479, 509, 514, 510, 537, 489, 515, 479, 497, 501, 517, 486, 538, 499, 492, 488, 524, 512, 521, 502, 534, 531, 474, 503, 513, 504, 506, 505, 505, 533, 550, 550, 514, 509, 503, 500, 488, 537, 512, 475, 495, 500, 543, 495, 530, 520, 529, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[476, 510, 570, 492, 524, 495, 520, 551, 518, 562, 515, 558, 507, 502, 524, 543, 503, 518, 501, 507, 539, 449, 484, 505, 545, 487, 502, 508, 506, 511, 482, 467, 501, 526, 516, 515, 550, 493, 544, 559, 508, 524, 506, 529, 528, 473, 487, 511, 476, 504, 522, 552, 517, 493, 507, 474, 509, 504, 484, 515, 510, 506, 504, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[548, 531, 549, 512, 498, 529, 507, 545, 0, 0, 0, 0, 0, 0, 0, 0, 497, 526, 511, 515, 515, 525, 487, 513, 0, 0, 0, 0, 0, 0, 0, 0, 503, 536, 525, 545, 496, 490, 506, 510, 27, 0, 0, 0, 0, 0, 0, 0, 541, 519, 510, 475, 495, 491, 507, 509, 5, 0, 0, 0, 0, 0, 0, 0, 526, 545, 513, 526, 480, 500, 469, 507, 0, 0, 0, 0, 0, 0, 0, 0, 509, 481, 524, 480, 479, 488, 523, 472, 8, 0, 0, 0, 0, 0, 0, 0, 522, 541, 521, 512, 521, 510, 504, 518, 42, 0, 0, 0, 0, 0, 0, 0, 481, 486, 506, 500, 491, 537, 530, 499, 16, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4219, 4089, 4138, 4052, 4066, 3964, 4191, 4049] avg=4096.0000 max=4219 max_rank=0 min=3964 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[478, 509, 539, 515, 522, 514, 482, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 474, 537, 502, 512, 487, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 488, 479, 508, 553, 508, 525, 509, 516, 24, 0, 0, 0, 0, 0, 0, 0, 480, 550, 516, 512, 544, 489, 563, 510, 5, 0, 0, 0, 0, 0, 0, 0, 525, 512, 525, 490, 525, 503, 543, 525, 0, 0, 0, 0, 0, 0, 0, 0, 495, 526, 521, 512, 490, 520, 510, 492, 8, 0, 0, 0, 0, 0, 0, 0, 506, 526, 528, 507, 519, 514, 491, 514, 42, 0, 0, 0, 0, 0, 0, 0, 523, 502, 454, 488, 497, 478, 522, 490, 16, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4051, 4110, 4169, 4148, 4074, 4147, 3972] avg=4096.0000 max=4169 max_rank=3 min=3972 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 511, 519, 524, 522, 525, 497, 508, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 469, 518, 533, 519, 521, 527, 0, 0, 0, 0, 0, 0, 0, 0, 473, 533, 521, 459, 501, 534, 517, 449, 26, 0, 0, 0, 0, 0, 0, 0, 501, 516, 533, 473, 528, 535, 494, 503, 5, 0, 0, 0, 0, 0, 0, 0, 505, 463, 550, 524, 506, 494, 490, 509, 0, 0, 0, 0, 0, 0, 0, 0, 489, 541, 502, 507, 501, 522, 493, 493, 8, 0, 0, 0, 0, 0, 0, 0, 497, 537, 555, 517, 519, 495, 521, 486, 43, 0, 0, 0, 0, 0, 0, 0, 492, 484, 543, 519, 520, 521, 510, 553, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4105, 4013, 4088, 4041, 4056, 4170, 4160] avg=4096.0000 max=4170 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 481, 527, 462, 508, 537, 501, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 461, 470, 511, 536, 533, 493, 0, 0, 0, 0, 0, 0, 0, 0, 530, 543, 501, 521, 518, 525, 503, 489, 27, 0, 0, 0, 0, 0, 0, 0, 521, 534, 517, 526, 546, 503, 507, 556, 5, 0, 0, 0, 0, 0, 0, 0, 485, 519, 511, 515, 514, 519, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 507, 518, 476, 527, 495, 497, 568, 532, 8, 0, 0, 0, 0, 0, 0, 0, 493, 512, 461, 514, 514, 495, 478, 510, 38, 0, 0, 0, 0, 0, 0, 0, 501, 563, 539, 516, 504, 478, 469, 535, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4006, 4024, 4157, 4215, 4100, 4128, 4015, 4123] avg=4096.0000 max=4215 max_rank=3 min=4006 min_rank=0 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[485, 486, 530, 499, 514, 477, 496, 488, 0, 0, 0, 0, 0, 0, 0, 0, 504, 517, 520, 514, 519, 493, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 514, 565, 496, 496, 502, 506, 505, 26, 0, 0, 0, 0, 0, 0, 0, 521, 491, 523, 493, 525, 541, 489, 517, 4, 0, 0, 0, 0, 0, 0, 0, 481, 546, 523, 508, 489, 538, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 528, 519, 511, 539, 514, 554, 516, 8, 0, 0, 0, 0, 0, 0, 0, 498, 516, 491, 508, 491, 496, 482, 537, 41, 0, 0, 0, 0, 0, 0, 0, 516, 498, 513, 488, 519, 506, 517, 486, 15, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4083, 4145, 4104, 4134, 4206, 4060, 4061] avg=4096.0000 max=4206 max_rank=5 min=3975 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 515, 542, 505, 522, 515, 544, 525, 0, 0, 0, 0, 0, 0, 0, 0, 512, 459, 529, 531, 478, 514, 542, 504, 0, 0, 0, 0, 0, 0, 0, 0, 479, 509, 514, 510, 537, 489, 515, 479, 23, 0, 0, 0, 0, 0, 0, 0, 497, 501, 517, 486, 538, 499, 492, 488, 5, 0, 0, 0, 0, 0, 0, 0, 496, 512, 521, 502, 534, 531, 474, 503, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 506, 505, 505, 533, 550, 550, 8, 0, 0, 0, 0, 0, 0, 0, 514, 509, 503, 500, 488, 537, 512, 475, 42, 0, 0, 0, 0, 0, 0, 0, 495, 500, 543, 495, 530, 520, 529, 485, 15, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4069, 4055, 4023, 4073, 4174, 4080, 4114] avg=4096.0000 max=4180 max_rank=0 min=4023 min_rank=3 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 524, 490, 487, 509, 533, 496, 475, 0, 0, 0, 0, 0, 0, 0, 0, 513, 483, 530, 508, 473, 556, 553, 532, 0, 0, 0, 0, 0, 0, 0, 0, 503, 489, 509, 511, 525, 541, 487, 522, 30, 0, 0, 0, 0, 0, 0, 0, 500, 499, 529, 515, 494, 501, 528, 531, 8, 0, 0, 0, 0, 0, 0, 0, 466, 535, 517, 480, 532, 493, 532, 473, 0, 0, 0, 0, 0, 0, 0, 0, 524, 503, 501, 525, 524, 488, 506, 510, 11, 0, 0, 0, 0, 0, 0, 0, 497, 485, 504, 494, 508, 468, 518, 499, 45, 5, 0, 0, 0, 0, 0, 0, 507, 551, 533, 482, 509, 518, 554, 521, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4055, 4148, 4117, 4105, 4028, 4092, 4023, 4200] avg=4096.0000 max=4200 max_rank=7 min=4023 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16343 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[476, 510, 570, 455, 524, 495, 520, 551, 0, 0, 0, 0, 0, 0, 0, 0, 518, 532, 515, 558, 507, 502, 524, 543, 0, 0, 0, 0, 0, 0, 0, 0, 503, 518, 501, 507, 539, 449, 484, 505, 27, 0, 0, 0, 0, 0, 0, 0, 545, 487, 502, 508, 506, 511, 482, 467, 4, 0, 0, 0, 0, 0, 0, 0, 475, 526, 516, 515, 550, 493, 544, 559, 0, 0, 0, 0, 0, 0, 0, 0, 508, 524, 506, 529, 528, 473, 487, 511, 8, 0, 0, 0, 0, 0, 0, 0, 476, 504, 522, 552, 517, 493, 507, 474, 37, 0, 0, 0, 0, 0, 0, 0, 509, 504, 484, 515, 510, 506, 504, 540, 14, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4199, 4033, 4012, 4178, 4074, 4082, 4089] avg=4096.0000 max=4199 max_rank=1 min=4012 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4352, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[533, 502, 536, 477, 515, 535, 493, 526, 534, 526, 522, 548, 480, 478, 495, 531, 501, 461, 493, 511, 557, 510, 471, 510, 500, 508, 493, 514, 508, 508, 516, 511, 510, 536, 553, 513, 479, 510, 511, 529, 513, 533, 485, 543, 538, 500, 492, 494, 486, 519, 493, 536, 513, 545, 526, 506, 520, 524, 464, 542, 497, 519, 514, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[540, 538, 493, 499, 533, 511, 515, 537, 486, 475, 520, 511, 523, 550, 519, 525, 509, 472, 503, 518, 505, 484, 525, 540, 506, 543, 488, 533, 479, 562, 526, 491, 501, 504, 495, 470, 511, 505, 537, 494, 492, 474, 528, 490, 545, 549, 515, 533, 485, 553, 520, 519, 503, 521, 515, 508, 486, 497, 483, 529, 524, 538, 488, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[510, 524, 506, 513, 489, 494, 518, 544, 499, 537, 509, 512, 562, 490, 506, 494, 527, 472, 536, 509, 537, 490, 525, 488, 487, 513, 515, 545, 522, 508, 526, 539, 484, 485, 532, 484, 517, 488, 531, 517, 493, 487, 502, 476, 540, 542, 528, 521, 525, 499, 522, 495, 484, 524, 526, 506, 524, 529, 517, 540, 521, 480, 486, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[559, 492, 511, 480, 529, 502, 486, 533, 486, 507, 494, 520, 506, 494, 512, 492, 472, 521, 544, 523, 545, 514, 493, 510, 533, 548, 514, 530, 515, 488, 471, 509, 531, 510, 537, 492, 481, 524, 535, 495, 511, 487, 503, 541, 499, 574, 532, 526, 516, 490, 491, 513, 546, 515, 508, 552, 537, 475, 468, 482, 515, 518, 497, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[508, 485, 535, 501, 546, 522, 488, 507, 489, 503, 550, 492, 506, 532, 517, 513, 519, 521, 508, 506, 530, 490, 544, 533, 513, 500, 511, 517, 513, 493, 480, 506, 531, 506, 548, 537, 523, 538, 519, 491, 503, 486, 511, 530, 485, 515, 519, 487, 501, 533, 517, 518, 525, 547, 511, 512, 490, 556, 494, 516, 472, 462, 524, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[557, 512, 506, 520, 510, 504, 556, 496, 561, 514, 512, 502, 511, 515, 509, 501, 491, 475, 451, 506, 514, 514, 494, 566, 491, 511, 516, 510, 496, 566, 478, 510, 482, 501, 512, 475, 537, 524, 514, 541, 527, 493, 510, 512, 515, 522, 533, 495, 485, 491, 507, 540, 513, 533, 498, 529, 504, 516, 539, 524, 543, 469, 502, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[489, 517, 519, 565, 541, 520, 508, 525, 542, 517, 487, 492, 492, 518, 508, 528, 548, 492, 504, 501, 525, 511, 530, 548, 536, 527, 553, 535, 494, 565, 532, 476, 528, 499, 526, 468, 528, 509, 493, 507, 536, 486, 506, 482, 474, 500, 511, 528, 476, 486, 489, 495, 524, 528, 516, 515, 476, 502, 488, 484, 513, 499, 504, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[534, 520, 487, 474, 529, 506, 525, 504, 550, 516, 535, 495, 529, 522, 518, 522, 487, 509, 466, 518, 514, 544, 497, 558, 522, 493, 487, 529, 510, 515, 521, 508, 541, 496, 489, 551, 474, 518, 517, 513, 517, 481, 507, 513, 519, 505, 493, 522, 509, 496, 488, 484, 511, 546, 473, 524, 503, 544, 537, 479, 496, 532, 529, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 538, 493, 499, 533, 511, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 478, 475, 520, 511, 523, 550, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0, 509, 472, 503, 518, 505, 484, 525, 540, 9, 0, 0, 0, 0, 0, 0, 0, 506, 543, 488, 523, 479, 562, 526, 491, 0, 0, 0, 0, 0, 0, 0, 0, 501, 504, 495, 470, 511, 505, 537, 494, 2, 1, 0, 0, 0, 0, 0, 0, 492, 474, 528, 490, 545, 549, 515, 533, 6, 0, 0, 0, 0, 0, 0, 0, 485, 553, 520, 519, 503, 520, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 486, 497, 483, 529, 524, 538, 488, 497, 28, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4101, 4065, 4118, 4020, 4132, 4123, 4071] avg=4096.0000 max=4138 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 524, 506, 513, 489, 494, 518, 544, 0, 0, 0, 0, 0, 0, 0, 0, 491, 537, 509, 512, 562, 490, 506, 494, 0, 0, 0, 0, 0, 0, 0, 0, 527, 472, 536, 509, 537, 490, 525, 488, 10, 0, 0, 0, 0, 0, 0, 0, 487, 513, 515, 534, 522, 508, 526, 539, 0, 0, 0, 0, 0, 0, 0, 0, 484, 485, 532, 484, 517, 488, 531, 517, 2, 1, 0, 0, 0, 0, 0, 0, 493, 487, 502, 476, 540, 542, 528, 521, 6, 0, 0, 0, 0, 0, 0, 0, 525, 499, 522, 495, 484, 523, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 529, 517, 540, 521, 480, 486, 517, 26, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4101, 4094, 4144, 4041, 4095, 4080, 4141] avg=4096.0000 max=4144 max_rank=3 min=4041 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 502, 536, 477, 515, 535, 493, 526, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 522, 548, 480, 478, 495, 531, 0, 0, 0, 0, 0, 0, 0, 0, 501, 461, 493, 511, 557, 510, 471, 510, 9, 0, 0, 0, 0, 0, 0, 0, 500, 508, 493, 504, 508, 508, 516, 511, 0, 0, 0, 0, 0, 0, 0, 0, 510, 536, 553, 513, 479, 510, 511, 529, 2, 1, 0, 0, 0, 0, 0, 0, 513, 533, 485, 543, 538, 500, 492, 494, 6, 0, 0, 0, 0, 0, 0, 0, 486, 519, 493, 536, 513, 544, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 464, 542, 497, 519, 514, 522, 27, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4106, 4023, 4048, 4144, 4104, 4123, 4130] avg=4096.0000 max=4144 max_rank=4 min=4023 min_rank=2 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 512, 506, 520, 510, 504, 556, 496, 0, 0, 0, 0, 0, 0, 0, 0, 552, 514, 512, 502, 511, 515, 509, 501, 0, 0, 0, 0, 0, 0, 0, 0, 491, 475, 451, 506, 514, 514, 494, 566, 9, 0, 0, 0, 0, 0, 0, 0, 491, 511, 516, 500, 496, 566, 478, 510, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 512, 475, 537, 524, 514, 541, 2, 1, 0, 0, 0, 0, 0, 0, 527, 493, 510, 512, 515, 522, 533, 495, 7, 0, 0, 0, 0, 0, 0, 0, 485, 491, 507, 540, 513, 532, 498, 529, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 539, 524, 543, 469, 502, 507, 29, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4116, 4020, 4068, 4089, 4114, 4095, 4134] avg=4096.0000 max=4134 max_rank=7 min=4020 min_rank=2 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[530, 492, 511, 480, 529, 502, 486, 533, 0, 0, 0, 0, 0, 0, 0, 0, 478, 507, 494, 520, 506, 494, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 472, 521, 544, 523, 545, 514, 493, 510, 9, 0, 0, 0, 0, 0, 0, 0, 533, 548, 514, 520, 515, 488, 471, 509, 0, 0, 0, 0, 0, 0, 0, 0, 531, 510, 537, 492, 481, 524, 535, 495, 2, 1, 0, 0, 0, 0, 0, 0, 511, 487, 503, 541, 499, 574, 532, 526, 6, 0, 0, 0, 0, 0, 0, 0, 516, 490, 491, 513, 546, 514, 508, 552, 0, 0, 0, 0, 0, 0, 0, 0, 537, 475, 468, 482, 515, 518, 497, 534, 29, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4003, 4131, 4098, 4108, 4179, 4130, 4056] avg=4096.0000 max=4179 max_rank=5 min=4003 min_rank=1 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 485, 535, 501, 546, 522, 488, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 503, 550, 492, 506, 532, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 519, 521, 508, 506, 530, 490, 544, 533, 9, 0, 0, 0, 0, 0, 0, 0, 513, 500, 511, 507, 513, 493, 480, 506, 0, 0, 0, 0, 0, 0, 0, 0, 531, 506, 548, 537, 523, 538, 519, 491, 2, 1, 0, 0, 0, 0, 0, 0, 503, 486, 511, 530, 485, 515, 519, 487, 6, 0, 0, 0, 0, 0, 0, 0, 501, 533, 517, 518, 525, 546, 511, 512, 0, 0, 0, 0, 0, 0, 0, 0, 490, 556, 494, 516, 472, 462, 524, 483, 26, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4094, 4160, 4023, 4196, 4042, 4163, 4024] avg=4096.0000 max=4196 max_rank=4 min=4023 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[464, 517, 519, 565, 541, 520, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 533, 517, 487, 492, 492, 518, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 548, 492, 504, 501, 525, 511, 530, 548, 9, 0, 0, 0, 0, 0, 0, 0, 536, 527, 553, 525, 494, 565, 532, 476, 0, 0, 0, 0, 0, 0, 0, 0, 528, 499, 526, 468, 528, 509, 493, 507, 2, 1, 0, 0, 0, 0, 0, 0, 536, 486, 506, 482, 474, 500, 511, 528, 7, 0, 0, 0, 0, 0, 0, 0, 476, 486, 489, 495, 524, 527, 516, 515, 0, 0, 0, 0, 0, 0, 0, 0, 476, 502, 488, 484, 513, 499, 504, 547, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4075, 4168, 4208, 4061, 4030, 4028, 4039] avg=4096.0000 max=4208 max_rank=3 min=4028 min_rank=6 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[503, 520, 487, 474, 529, 506, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 535, 495, 529, 522, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 487, 509, 466, 518, 514, 544, 497, 558, 14, 0, 0, 0, 0, 0, 0, 0, 522, 493, 487, 505, 510, 515, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 541, 496, 489, 551, 474, 518, 517, 513, 5, 5, 0, 0, 0, 0, 0, 0, 517, 481, 507, 513, 519, 505, 493, 522, 10, 5, 0, 0, 0, 0, 0, 0, 509, 496, 488, 484, 511, 541, 473, 524, 0, 0, 0, 0, 0, 0, 0, 0, 503, 544, 537, 479, 496, 532, 529, 517, 31, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4048, 4172, 4107, 4061, 4109, 4072, 4026, 4173] avg=4096.0000 max=4173 max_rank=7 min=4026 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4096, 4096, 4224, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4352, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4096, 4352, 4096, 4096, 4352, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[522, 506, 487, 531, 487, 521, 537, 511, 483, 525, 478, 520, 535, 484, 524, 477, 481, 547, 510, 506, 500, 508, 541, 514, 491, 472, 510, 549, 512, 518, 512, 548, 501, 523, 500, 485, 509, 513, 536, 491, 506, 510, 530, 475, 537, 528, 510, 518, 528, 521, 517, 531, 530, 501, 538, 498, 517, 489, 511, 524, 533, 492, 521, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[480, 545, 493, 530, 525, 534, 510, 554, 553, 506, 525, 497, 498, 489, 502, 526, 496, 516, 538, 497, 557, 526, 467, 517, 503, 485, 521, 520, 512, 518, 511, 500, 505, 523, 541, 498, 507, 548, 528, 524, 525, 485, 528, 543, 533, 491, 462, 496, 480, 532, 513, 498, 498, 525, 539, 547, 491, 493, 476, 505, 457, 508, 487, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[540, 526, 532, 498, 522, 546, 521, 535, 486, 535, 499, 511, 530, 521, 521, 498, 504, 506, 531, 527, 555, 526, 484, 510, 534, 489, 528, 527, 489, 464, 485, 526, 532, 530, 476, 495, 510, 470, 532, 508, 531, 471, 485, 522, 511, 469, 514, 528, 548, 486, 481, 517, 499, 515, 516, 493, 506, 524, 532, 528, 495, 512, 511, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[482, 505, 500, 500, 513, 523, 492, 502, 511, 513, 533, 490, 555, 476, 510, 498, 524, 512, 536, 548, 505, 518, 502, 511, 511, 518, 529, 572, 545, 464, 512, 544, 533, 482, 527, 502, 494, 508, 512, 543, 496, 503, 514, 516, 484, 492, 513, 519, 522, 511, 485, 488, 530, 465, 531, 495, 555, 499, 497, 549, 509, 507, 499, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[510, 500, 514, 497, 522, 510, 520, 489, 486, 505, 504, 480, 521, 552, 526, 545, 537, 503, 536, 467, 489, 528, 484, 508, 518, 520, 515, 515, 540, 487, 517, 514, 525, 522, 533, 483, 506, 490, 497, 524, 515, 538, 528, 550, 511, 506, 509, 521, 462, 553, 488, 509, 471, 531, 494, 502, 519, 509, 532, 494, 548, 504, 525, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[522, 502, 496, 536, 551, 490, 516, 512, 519, 552, 509, 526, 527, 520, 537, 533, 488, 519, 557, 480, 521, 506, 498, 525, 497, 480, 494, 517, 562, 470, 504, 505, 497, 547, 508, 502, 512, 482, 476, 483, 516, 533, 505, 508, 535, 497, 517, 530, 523, 513, 511, 466, 504, 488, 513, 519, 490, 535, 487, 528, 514, 518, 516, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[523, 525, 525, 516, 461, 560, 521, 522, 503, 502, 518, 503, 538, 508, 501, 513, 511, 523, 515, 538, 511, 492, 552, 513, 539, 528, 537, 471, 521, 487, 524, 485, 503, 513, 524, 468, 517, 513, 494, 488, 536, 470, 530, 520, 492, 507, 499, 561, 496, 543, 492, 532, 504, 532, 503, 528, 470, 504, 508, 492, 532, 491, 511, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[514, 537, 524, 517, 500, 507, 513, 504, 503, 501, 486, 500, 521, 499, 550, 479, 489, 504, 511, 544, 507, 497, 513, 475, 458, 517, 507, 492, 489, 527, 554, 483, 529, 513, 510, 524, 486, 488, 533, 515, 539, 535, 505, 524, 516, 506, 561, 505, 541, 514, 498, 506, 531, 534, 514, 535, 488, 549, 508, 500, 519, 494, 538, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 505, 500, 500, 513, 499, 492, 502, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 533, 490, 554, 476, 510, 498, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 525, 548, 505, 518, 502, 511, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 529, 572, 545, 464, 512, 544, 7, 1, 0, 0, 0, 0, 0, 0, 533, 482, 527, 502, 494, 508, 512, 543, 22, 0, 0, 0, 0, 0, 0, 0, 496, 503, 514, 516, 484, 492, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 522, 511, 485, 488, 530, 465, 531, 495, 4, 1, 0, 0, 0, 0, 0, 0, 555, 499, 497, 549, 509, 507, 499, 534, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4085, 4145, 4203, 4123, 4026, 4032, 4161] avg=4096.0000 max=4203 max_rank=3 min=3993 min_rank=0 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[540, 526, 532, 498, 522, 521, 521, 535, 0, 0, 0, 0, 0, 0, 0, 0, 486, 535, 499, 511, 529, 521, 521, 498, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 520, 527, 555, 526, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 489, 528, 527, 489, 464, 485, 526, 7, 1, 0, 0, 0, 0, 0, 0, 532, 530, 476, 495, 510, 470, 532, 508, 23, 0, 0, 0, 0, 0, 0, 0, 531, 471, 485, 522, 511, 469, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 548, 486, 481, 517, 499, 515, 516, 493, 4, 1, 0, 0, 0, 0, 0, 0, 506, 524, 532, 528, 495, 512, 511, 515, 11, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4100, 4132, 4050, 4076, 4019, 4060, 4136] avg=4096.0000 max=4195 max_rank=0 min=4019 min_rank=5 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16291 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 500, 514, 497, 522, 487, 520, 489, 0, 0, 0, 0, 0, 0, 0, 0, 486, 505, 504, 480, 521, 552, 526, 545, 0, 0, 0, 0, 0, 0, 0, 0, 537, 503, 525, 467, 489, 528, 484, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 515, 515, 540, 487, 517, 514, 7, 1, 0, 0, 0, 0, 0, 0, 525, 522, 533, 483, 506, 490, 497, 524, 21, 0, 0, 0, 0, 0, 0, 0, 515, 538, 528, 550, 511, 506, 509, 510, 0, 0, 0, 0, 0, 0, 0, 0, 462, 553, 488, 509, 471, 531, 494, 502, 4, 0, 0, 0, 0, 0, 0, 0, 519, 509, 532, 494, 548, 504, 525, 510, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4119, 4041, 4134, 4101, 4167, 4014, 4153] avg=4096.0000 max=4167 max_rank=5 min=4014 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[480, 545, 493, 530, 525, 510, 510, 554, 0, 0, 0, 0, 0, 0, 0, 0, 553, 506, 525, 497, 498, 489, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 527, 497, 557, 526, 467, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 485, 521, 520, 512, 518, 511, 500, 7, 1, 0, 0, 0, 0, 0, 0, 505, 523, 541, 498, 507, 548, 528, 524, 22, 0, 0, 0, 0, 0, 0, 0, 525, 485, 528, 543, 533, 491, 462, 485, 0, 0, 0, 0, 0, 0, 0, 0, 480, 532, 513, 498, 498, 525, 539, 547, 4, 0, 0, 0, 0, 0, 0, 0, 491, 493, 476, 505, 457, 508, 487, 531, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4147, 4096, 4103, 4078, 4196, 4052, 4136, 3960] avg=4096.0000 max=4196 max_rank=4 min=3960 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 506, 487, 531, 487, 497, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 483, 525, 478, 520, 534, 484, 524, 477, 0, 0, 0, 0, 0, 0, 0, 0, 481, 547, 500, 506, 500, 508, 541, 514, 0, 0, 0, 0, 0, 0, 0, 0, 491, 472, 510, 549, 512, 518, 512, 548, 7, 1, 0, 0, 0, 0, 0, 0, 501, 523, 500, 485, 509, 513, 536, 491, 22, 0, 0, 0, 0, 0, 0, 0, 506, 510, 530, 475, 537, 528, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 528, 521, 517, 531, 530, 501, 538, 498, 3, 1, 0, 0, 0, 0, 0, 0, 517, 489, 511, 524, 533, 492, 521, 498, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4025, 4097, 4120, 4080, 4103, 4168, 4097] avg=4096.0000 max=4168 max_rank=6 min=4025 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 502, 496, 536, 551, 468, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 519, 552, 509, 526, 527, 520, 537, 533, 0, 0, 0, 0, 0, 0, 0, 0, 488, 519, 545, 480, 521, 506, 498, 525, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 494, 517, 562, 470, 504, 505, 8, 1, 0, 0, 0, 0, 0, 0, 497, 547, 508, 502, 512, 482, 476, 483, 20, 0, 0, 0, 0, 0, 0, 0, 516, 533, 505, 508, 535, 497, 517, 518, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 511, 466, 504, 488, 513, 519, 4, 0, 0, 0, 0, 0, 0, 0, 490, 535, 487, 528, 514, 518, 516, 524, 11, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4223, 4082, 4038, 4027, 4129, 4041, 4125] avg=4096.0000 max=4223 max_rank=1 min=4027 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 525, 525, 516, 461, 527, 521, 522, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 518, 503, 533, 508, 501, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 498, 538, 511, 492, 552, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 528, 537, 471, 521, 487, 524, 485, 11, 7, 0, 0, 0, 0, 0, 0, 503, 513, 524, 468, 517, 513, 494, 488, 26, 0, 0, 0, 0, 0, 0, 0, 536, 470, 530, 520, 492, 507, 499, 538, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 492, 532, 504, 532, 503, 528, 6, 5, 0, 0, 0, 0, 0, 0, 470, 504, 508, 492, 532, 491, 511, 509, 16, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4081, 4138, 4110, 4046, 4092, 4141, 4040] avg=4096.0000 max=4141 max_rank=6 min=4040 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 537, 524, 517, 500, 484, 513, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 501, 486, 500, 521, 499, 550, 479, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 501, 544, 507, 497, 513, 475, 0, 0, 0, 0, 0, 0, 0, 0, 458, 517, 507, 492, 489, 527, 554, 483, 7, 1, 0, 0, 0, 0, 0, 0, 529, 513, 510, 524, 486, 488, 533, 515, 21, 0, 0, 0, 0, 0, 0, 0, 539, 535, 505, 524, 516, 506, 561, 494, 0, 0, 0, 0, 0, 0, 0, 0, 541, 514, 498, 506, 531, 534, 514, 535, 3, 0, 0, 0, 0, 0, 0, 0, 488, 549, 508, 500, 519, 494, 538, 488, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4039, 4030, 4035, 4119, 4180, 4176, 4096] avg=4096.0000 max=4180 max_rank=5 min=4030 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 3968, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[534, 486, 510, 517, 501, 515, 483, 519, 513, 516, 514, 523, 500, 494, 541, 513, 504, 528, 468, 469, 510, 503, 539, 528, 524, 533, 518, 515, 501, 538, 548, 483, 482, 507, 519, 466, 536, 508, 539, 480, 511, 502, 488, 512, 506, 499, 513, 548, 509, 556, 473, 531, 530, 497, 545, 510, 523, 501, 486, 520, 505, 524, 531, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[512, 484, 501, 516, 490, 530, 562, 509, 479, 499, 474, 527, 520, 517, 502, 504, 504, 545, 514, 491, 531, 517, 526, 498, 491, 526, 526, 509, 513, 539, 542, 519, 519, 499, 518, 518, 489, 515, 574, 560, 468, 530, 505, 500, 509, 520, 519, 493, 505, 538, 497, 503, 510, 502, 511, 492, 532, 509, 492, 497, 494, 511, 490, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[528, 515, 517, 532, 508, 488, 504, 506, 530, 516, 506, 577, 507, 531, 504, 517, 525, 515, 524, 490, 521, 502, 525, 495, 471, 497, 512, 544, 532, 527, 517, 550, 500, 518, 542, 515, 524, 466, 493, 505, 525, 492, 478, 532, 471, 510, 472, 503, 511, 506, 499, 491, 524, 512, 514, 524, 550, 505, 501, 502, 514, 529, 506, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[471, 477, 510, 506, 491, 481, 561, 494, 502, 502, 522, 518, 483, 537, 502, 525, 504, 522, 479, 509, 499, 493, 496, 549, 536, 514, 495, 480, 516, 512, 533, 502, 510, 531, 541, 449, 535, 501, 483, 532, 499, 536, 518, 481, 505, 505, 536, 486, 540, 497, 546, 538, 528, 553, 507, 526, 527, 535, 532, 501, 533, 509, 534, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[501, 508, 507, 508, 491, 512, 513, 483, 480, 552, 512, 573, 525, 530, 514, 513, 526, 523, 531, 513, 500, 477, 526, 504, 517, 530, 506, 479, 529, 515, 510, 477, 545, 519, 476, 506, 524, 498, 520, 523, 509, 520, 547, 505, 515, 506, 523, 507, 528, 507, 524, 475, 510, 578, 499, 462, 530, 471, 473, 485, 526, 542, 533, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[495, 521, 543, 531, 531, 526, 522, 503, 508, 502, 529, 538, 496, 562, 530, 489, 530, 557, 537, 472, 528, 494, 531, 541, 503, 538, 525, 516, 528, 538, 530, 489, 502, 444, 493, 527, 524, 490, 544, 503, 534, 469, 504, 485, 538, 475, 467, 513, 471, 531, 505, 472, 545, 481, 493, 469, 505, 536, 472, 490, 511, 523, 538, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[542, 509, 519, 514, 493, 494, 511, 495, 513, 511, 492, 494, 544, 504, 517, 547, 495, 519, 494, 506, 450, 518, 513, 526, 555, 551, 481, 485, 509, 487, 488, 516, 493, 480, 531, 545, 551, 528, 510, 551, 509, 571, 525, 508, 492, 534, 520, 512, 503, 504, 473, 466, 535, 501, 535, 486, 518, 522, 520, 500, 558, 510, 489, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[518, 530, 517, 513, 540, 502, 517, 535, 523, 542, 506, 511, 530, 487, 517, 522, 543, 459, 492, 473, 497, 550, 489, 533, 490, 533, 520, 500, 518, 533, 511, 484, 532, 485, 515, 520, 528, 486, 527, 489, 545, 549, 520, 489, 517, 481, 517, 504, 503, 533, 475, 544, 521, 483, 513, 498, 511, 511, 508, 482, 490, 528, 524, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[534, 486, 510, 517, 501, 515, 483, 519, 5, 1, 1, 0, 0, 0, 0, 0, 513, 516, 514, 492, 500, 494, 541, 513, 0, 0, 0, 0, 0, 0, 0, 0, 504, 528, 468, 469, 510, 503, 539, 528, 12, 0, 0, 0, 0, 0, 0, 0, 524, 511, 518, 515, 501, 538, 548, 483, 0, 0, 0, 0, 0, 0, 0, 0, 482, 507, 519, 466, 535, 508, 539, 480, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 488, 512, 506, 499, 513, 548, 22, 0, 0, 0, 0, 0, 0, 0, 509, 556, 473, 531, 530, 497, 545, 510, 9, 5, 0, 0, 0, 0, 0, 0, 522, 501, 486, 520, 505, 524, 531, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4083, 4061, 4138, 4036, 4101, 4165, 4112] avg=4096.0000 max=4165 max_rank=6 min=4036 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[528, 515, 517, 532, 508, 488, 504, 506, 4, 1, 1, 0, 0, 0, 0, 0, 530, 516, 506, 542, 507, 531, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 515, 524, 490, 521, 502, 525, 495, 11, 0, 0, 0, 0, 0, 0, 0, 471, 477, 512, 544, 532, 527, 517, 550, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 542, 515, 523, 466, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 492, 478, 532, 471, 510, 472, 503, 25, 0, 0, 0, 0, 0, 0, 0, 511, 506, 499, 491, 524, 512, 514, 524, 10, 5, 0, 0, 0, 0, 0, 0, 549, 505, 501, 502, 514, 529, 506, 501, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4153, 4108, 4130, 4062, 4008, 4096, 4107] avg=4096.0000 max=4153 max_rank=1 min=4008 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16273 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[471, 477, 510, 506, 491, 481, 561, 494, 5, 1, 1, 0, 0, 0, 0, 0, 502, 502, 522, 488, 483, 537, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 504, 522, 479, 509, 499, 493, 496, 549, 11, 0, 0, 0, 0, 0, 0, 0, 536, 493, 495, 480, 516, 512, 533, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 531, 541, 449, 534, 501, 483, 532, 0, 0, 0, 0, 0, 0, 0, 0, 499, 536, 518, 481, 505, 505, 536, 486, 22, 0, 0, 0, 0, 0, 0, 0, 540, 497, 546, 538, 528, 553, 507, 526, 8, 5, 0, 0, 0, 0, 0, 0, 526, 535, 532, 501, 533, 509, 534, 493, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4061, 4062, 4067, 4081, 4088, 4248, 4163] avg=4096.0000 max=4248 max_rank=6 min=3998 min_rank=0 imbalance_ratio=1.037109 max_load_reduction=28520 max_load_reduction_pct=0.870361 moved_to_echo=16580 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 484, 501, 516, 490, 530, 562, 509, 5, 1, 1, 0, 0, 0, 0, 0, 479, 499, 474, 495, 520, 517, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 545, 514, 491, 531, 517, 526, 498, 12, 0, 0, 0, 0, 0, 0, 0, 491, 504, 526, 509, 513, 539, 542, 519, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 518, 518, 488, 515, 574, 560, 0, 0, 0, 0, 0, 0, 0, 0, 468, 530, 505, 500, 509, 520, 519, 493, 23, 0, 0, 0, 0, 0, 0, 0, 505, 538, 497, 503, 510, 502, 511, 492, 9, 5, 0, 0, 0, 0, 0, 0, 531, 509, 492, 497, 494, 511, 490, 532, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 3990, 4138, 4143, 4191, 4067, 4072, 4056] avg=4096.0000 max=4191 max_rank=4 min=3990 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[501, 508, 507, 508, 491, 512, 513, 483, 5, 1, 1, 0, 0, 0, 0, 0, 480, 552, 512, 539, 525, 530, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 531, 513, 500, 477, 526, 504, 12, 0, 0, 0, 0, 0, 0, 0, 517, 508, 506, 479, 529, 515, 510, 477, 0, 0, 0, 0, 0, 0, 0, 0, 545, 519, 476, 506, 523, 498, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 509, 520, 547, 505, 515, 506, 523, 507, 25, 0, 0, 0, 0, 0, 0, 0, 528, 507, 524, 475, 510, 578, 499, 462, 9, 5, 0, 0, 0, 0, 0, 0, 529, 471, 473, 485, 526, 542, 533, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4165, 4112, 4041, 4110, 4157, 4097, 4056] avg=4096.0000 max=4165 max_rank=1 min=4030 min_rank=0 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[542, 509, 519, 514, 493, 494, 511, 495, 5, 1, 1, 0, 0, 0, 0, 0, 513, 511, 492, 465, 544, 504, 517, 547, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 494, 506, 450, 518, 513, 526, 12, 0, 0, 0, 0, 0, 0, 0, 555, 528, 481, 485, 509, 487, 488, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 480, 531, 545, 550, 528, 510, 551, 0, 0, 0, 0, 0, 0, 0, 0, 509, 571, 525, 508, 492, 534, 520, 512, 21, 0, 0, 0, 0, 0, 0, 0, 503, 504, 473, 466, 535, 501, 535, 486, 8, 6, 0, 0, 0, 0, 0, 0, 517, 522, 520, 500, 558, 510, 489, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4093, 4033, 4049, 4188, 4192, 4017, 4112] avg=4096.0000 max=4192 max_rank=5 min=4017 min_rank=6 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 530, 517, 513, 540, 502, 517, 535, 5, 1, 1, 0, 0, 0, 0, 0, 523, 542, 506, 481, 530, 487, 517, 522, 0, 0, 0, 0, 0, 0, 0, 0, 543, 459, 492, 473, 497, 550, 489, 533, 12, 0, 0, 0, 0, 0, 0, 0, 490, 511, 520, 500, 518, 533, 511, 484, 0, 0, 0, 0, 0, 0, 0, 0, 532, 485, 515, 520, 527, 486, 527, 489, 0, 0, 0, 0, 0, 0, 0, 0, 545, 549, 520, 489, 517, 481, 517, 504, 22, 0, 0, 0, 0, 0, 0, 0, 503, 533, 475, 544, 521, 483, 513, 498, 8, 5, 0, 0, 0, 0, 0, 0, 510, 511, 508, 482, 490, 528, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4179, 4108, 4048, 4067, 4081, 4144, 4083, 4058] avg=4096.0000 max=4179 max_rank=0 min=4048 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[495, 521, 543, 531, 531, 526, 522, 503, 8, 8, 2, 0, 0, 0, 0, 0, 508, 502, 529, 499, 496, 562, 530, 489, 0, 0, 0, 0, 0, 0, 0, 0, 530, 557, 537, 472, 528, 494, 531, 541, 16, 0, 0, 0, 0, 0, 0, 0, 503, 504, 525, 516, 528, 538, 530, 489, 0, 0, 0, 0, 0, 0, 0, 0, 502, 444, 493, 527, 516, 490, 544, 503, 0, 0, 0, 0, 0, 0, 0, 0, 534, 469, 504, 485, 538, 475, 467, 513, 26, 0, 0, 0, 0, 0, 0, 0, 471, 531, 505, 472, 545, 481, 493, 469, 13, 10, 0, 0, 0, 0, 0, 0, 503, 536, 472, 490, 511, 523, 538, 531, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4115, 4206, 4133, 4019, 4011, 3990, 4104] avg=4096.0000 max=4206 max_rank=2 min=3990 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16124 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 3968, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[496, 537, 511, 492, 465, 495, 487, 514, 535, 509, 551, 534, 531, 514, 480, 527, 509, 519, 563, 520, 560, 442, 530, 516, 525, 521, 488, 494, 491, 531, 515, 522, 518, 510, 507, 486, 515, 540, 496, 505, 535, 498, 498, 504, 485, 486, 480, 514, 511, 497, 556, 532, 524, 569, 525, 526, 492, 514, 491, 501, 516, 501, 518, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[527, 488, 537, 469, 545, 498, 495, 519, 506, 516, 521, 512, 515, 518, 491, 482, 505, 521, 518, 512, 536, 545, 521, 528, 507, 489, 500, 518, 561, 537, 532, 495, 496, 529, 496, 516, 487, 513, 501, 506, 543, 486, 516, 527, 518, 503, 446, 535, 524, 539, 501, 529, 523, 544, 475, 526, 511, 494, 509, 512, 516, 473, 509, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[505, 540, 506, 494, 519, 479, 534, 527, 509, 500, 514, 500, 515, 490, 509, 496, 547, 514, 518, 542, 517, 528, 506, 481, 510, 514, 516, 502, 510, 519, 517, 500, 513, 524, 494, 459, 482, 517, 525, 457, 531, 520, 512, 555, 563, 474, 522, 529, 498, 525, 518, 532, 530, 522, 461, 497, 477, 509, 534, 520, 532, 543, 502, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[486, 494, 556, 512, 520, 508, 468, 522, 521, 559, 540, 492, 495, 512, 483, 548, 535, 528, 522, 507, 512, 523, 520, 500, 500, 485, 538, 537, 497, 513, 502, 533, 512, 477, 510, 515, 509, 510, 523, 517, 545, 475, 522, 512, 492, 519, 516, 498, 478, 517, 516, 487, 508, 558, 497, 490, 479, 545, 473, 515, 516, 518, 538, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[515, 524, 514, 504, 550, 491, 539, 494, 516, 522, 539, 553, 502, 521, 507, 474, 514, 480, 505, 501, 526, 530, 503, 509, 551, 528, 507, 495, 473, 464, 522, 480, 516, 531, 514, 510, 528, 490, 527, 504, 518, 549, 484, 523, 511, 517, 520, 515, 507, 500, 512, 515, 510, 514, 510, 484, 506, 459, 557, 516, 512, 521, 517, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[458, 496, 553, 513, 532, 531, 491, 524, 503, 482, 514, 501, 510, 548, 517, 503, 492, 513, 516, 525, 527, 499, 544, 559, 502, 516, 527, 496, 527, 513, 502, 495, 503, 497, 517, 518, 505, 490, 489, 491, 504, 490, 515, 509, 519, 496, 541, 483, 489, 534, 507, 530, 523, 556, 518, 539, 527, 506, 493, 523, 526, 499, 491, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[499, 531, 502, 528, 511, 528, 530, 507, 477, 488, 530, 504, 553, 530, 514, 524, 523, 510, 512, 542, 494, 515, 564, 512, 510, 526, 531, 511, 517, 514, 532, 495, 495, 480, 520, 510, 542, 499, 518, 518, 496, 475, 522, 503, 476, 482, 520, 506, 545, 529, 474, 511, 547, 493, 481, 528, 477, 521, 495, 512, 511, 487, 536, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[534, 518, 518, 523, 478, 511, 522, 502, 565, 517, 485, 547, 519, 515, 494, 516, 512, 475, 506, 502, 534, 502, 525, 546, 529, 528, 526, 515, 500, 501, 560, 566, 480, 528, 514, 520, 524, 526, 520, 479, 459, 487, 506, 506, 508, 508, 498, 514, 501, 506, 524, 483, 536, 537, 493, 465, 508, 512, 521, 506, 519, 486, 508, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[496, 537, 511, 492, 465, 495, 487, 514, 6, 0, 0, 0, 0, 0, 0, 0, 535, 509, 532, 534, 531, 514, 480, 527, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 563, 520, 560, 442, 482, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 521, 488, 494, 491, 531, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 507, 486, 515, 540, 496, 505, 37, 0, 0, 0, 0, 0, 0, 0, 535, 498, 498, 504, 485, 486, 480, 514, 20, 11, 0, 0, 0, 0, 0, 0, 511, 497, 556, 532, 524, 548, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 492, 514, 491, 501, 516, 501, 518, 494, 19, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4162, 4111, 4074, 4114, 4031, 4219, 4054] avg=4096.0000 max=4219 max_rank=6 min=4003 min_rank=0 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[458, 496, 553, 513, 532, 531, 491, 524, 6, 0, 0, 0, 0, 0, 0, 0, 503, 482, 496, 501, 510, 548, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 492, 513, 516, 525, 527, 499, 494, 559, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 527, 496, 527, 513, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 517, 518, 505, 490, 489, 491, 38, 0, 0, 0, 0, 0, 0, 0, 504, 490, 515, 509, 519, 496, 541, 483, 20, 12, 0, 0, 0, 0, 0, 0, 489, 534, 507, 530, 523, 535, 518, 539, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 493, 523, 526, 499, 491, 511, 18, 6, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4060, 4125, 4066, 4048, 4089, 4175, 4101] avg=4096.0000 max=4175 max_rank=6 min=4048 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 524, 514, 504, 550, 491, 539, 494, 6, 0, 0, 0, 0, 0, 0, 0, 516, 522, 521, 553, 502, 521, 507, 474, 0, 0, 0, 0, 0, 0, 0, 0, 514, 480, 505, 501, 526, 530, 457, 509, 0, 0, 0, 0, 0, 0, 0, 0, 551, 528, 507, 495, 473, 464, 509, 480, 0, 0, 0, 0, 0, 0, 0, 0, 516, 531, 514, 510, 528, 490, 527, 504, 35, 0, 0, 0, 0, 0, 0, 0, 518, 549, 484, 523, 511, 517, 520, 515, 18, 11, 0, 0, 0, 0, 0, 0, 507, 500, 512, 515, 510, 495, 510, 484, 0, 0, 0, 0, 0, 0, 0, 0, 506, 459, 557, 516, 512, 521, 517, 518, 18, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4116, 4022, 4007, 4155, 4166, 4033, 4132] avg=4096.0000 max=4166 max_rank=5 min=4007 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 494, 556, 512, 520, 508, 468, 522, 6, 0, 0, 0, 0, 0, 0, 0, 521, 559, 522, 492, 495, 512, 483, 548, 0, 0, 0, 0, 0, 0, 0, 0, 535, 528, 522, 507, 512, 523, 472, 500, 0, 0, 0, 0, 0, 0, 0, 0, 500, 485, 538, 537, 497, 513, 490, 533, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 510, 515, 509, 510, 523, 517, 37, 0, 0, 0, 0, 0, 0, 0, 545, 475, 522, 512, 492, 519, 516, 498, 20, 11, 0, 0, 0, 0, 0, 0, 478, 517, 516, 487, 508, 537, 497, 490, 0, 0, 0, 0, 0, 0, 0, 0, 479, 545, 473, 515, 516, 518, 538, 513, 18, 6, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4132, 4099, 4093, 4110, 4110, 4030, 4122] avg=4096.0000 max=4132 max_rank=1 min=4030 min_rank=6 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 488, 537, 469, 545, 498, 495, 519, 6, 0, 0, 0, 0, 0, 0, 0, 506, 516, 503, 512, 515, 518, 491, 482, 0, 0, 0, 0, 0, 0, 0, 0, 505, 521, 518, 512, 536, 545, 473, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 489, 500, 518, 561, 537, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 496, 529, 496, 516, 487, 513, 501, 506, 37, 0, 0, 0, 0, 0, 0, 0, 543, 486, 516, 527, 518, 503, 446, 535, 19, 11, 0, 0, 0, 0, 0, 0, 524, 539, 501, 529, 523, 524, 475, 526, 0, 0, 0, 0, 0, 0, 0, 0, 511, 494, 509, 512, 516, 473, 509, 501, 18, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4043, 4138, 4126, 4081, 4104, 4141, 4051] avg=4096.0000 max=4141 max_rank=6 min=4043 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 540, 506, 494, 519, 479, 534, 527, 6, 0, 0, 0, 0, 0, 0, 0, 509, 500, 496, 500, 515, 490, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 547, 514, 518, 542, 517, 528, 459, 481, 0, 0, 0, 0, 0, 0, 0, 0, 510, 514, 516, 502, 510, 519, 504, 500, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 494, 459, 482, 517, 525, 457, 36, 0, 0, 0, 0, 0, 0, 0, 531, 520, 512, 555, 563, 474, 522, 529, 18, 11, 0, 0, 0, 0, 0, 0, 498, 525, 518, 532, 530, 503, 461, 497, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 534, 520, 532, 543, 502, 513, 18, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4015, 4106, 4075, 4007, 4235, 4064, 4156] avg=4096.0000 max=4235 max_rank=5 min=4007 min_rank=4 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 531, 502, 528, 511, 528, 530, 507, 6, 0, 0, 0, 0, 0, 0, 0, 477, 488, 512, 504, 553, 530, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 512, 542, 494, 515, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 531, 511, 517, 514, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 495, 480, 520, 510, 542, 499, 518, 518, 40, 0, 0, 0, 0, 0, 0, 0, 496, 475, 522, 503, 476, 482, 520, 506, 17, 12, 0, 0, 0, 0, 0, 0, 545, 529, 474, 511, 547, 475, 481, 528, 0, 0, 0, 0, 0, 0, 0, 0, 477, 521, 495, 512, 511, 487, 536, 495, 18, 7, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4102, 4120, 4123, 4122, 4009, 4090, 4060] avg=4096.0000 max=4142 max_rank=0 min=4009 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 518, 518, 523, 478, 511, 522, 502, 10, 0, 0, 0, 0, 0, 0, 0, 565, 517, 465, 547, 519, 515, 494, 516, 0, 0, 0, 0, 0, 0, 0, 0, 512, 475, 506, 502, 534, 502, 470, 546, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 526, 515, 500, 501, 539, 566, 0, 0, 0, 0, 0, 0, 0, 0, 480, 528, 514, 520, 524, 526, 520, 479, 40, 0, 0, 0, 0, 0, 0, 0, 459, 487, 506, 506, 508, 508, 498, 514, 23, 15, 0, 0, 0, 0, 0, 0, 501, 506, 524, 483, 536, 508, 493, 465, 0, 0, 0, 0, 0, 0, 0, 0, 508, 512, 521, 506, 519, 486, 508, 495, 20, 11, 6, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4138, 4047, 4204, 4131, 4024, 4016, 4092] avg=4096.0000 max=4204 max_rank=3 min=4016 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[516, 518, 532, 481, 522, 522, 495, 515, 553, 497, 526, 507, 558, 518, 478, 476, 531, 500, 539, 498, 496, 521, 491, 479, 489, 493, 546, 489, 540, 528, 520, 513, 509, 519, 475, 503, 529, 477, 575, 558, 521, 498, 512, 494, 546, 522, 535, 472, 496, 537, 483, 490, 493, 498, 517, 469, 538, 496, 524, 534, 530, 499, 514, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[522, 499, 498, 515, 503, 508, 527, 537, 502, 497, 510, 532, 525, 503, 487, 513, 508, 529, 502, 510, 566, 522, 524, 500, 487, 507, 495, 533, 504, 495, 525, 525, 526, 526, 497, 517, 503, 517, 521, 513, 510, 525, 550, 542, 489, 507, 488, 519, 486, 515, 502, 526, 497, 485, 538, 486, 520, 518, 479, 539, 517, 519, 478, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[514, 532, 531, 520, 495, 462, 518, 532, 478, 490, 521, 565, 523, 496, 519, 544, 520, 506, 485, 528, 507, 511, 523, 489, 524, 525, 574, 499, 520, 511, 540, 455, 512, 489, 472, 509, 510, 527, 499, 511, 503, 521, 523, 504, 519, 497, 526, 497, 505, 483, 528, 514, 516, 532, 522, 533, 501, 483, 496, 490, 511, 553, 513, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[511, 554, 545, 534, 531, 548, 499, 520, 515, 501, 519, 526, 513, 482, 540, 516, 495, 494, 505, 529, 492, 494, 532, 497, 509, 520, 513, 500, 506, 518, 544, 517, 519, 474, 525, 522, 467, 512, 533, 510, 476, 504, 502, 514, 538, 537, 525, 517, 505, 499, 510, 471, 497, 466, 519, 505, 459, 489, 532, 535, 508, 510, 570, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[514, 581, 501, 520, 519, 497, 512, 531, 478, 519, 544, 503, 548, 473, 550, 527, 521, 529, 478, 528, 493, 513, 479, 507, 520, 490, 522, 552, 496, 522, 514, 496, 487, 487, 527, 504, 538, 478, 524, 502, 500, 526, 487, 536, 516, 507, 500, 493, 513, 469, 537, 508, 526, 539, 510, 531, 508, 509, 518, 509, 470, 489, 507, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[549, 512, 540, 501, 510, 527, 548, 493, 496, 545, 512, 532, 505, 548, 498, 519, 499, 495, 508, 501, 516, 492, 522, 497, 532, 527, 509, 509, 485, 522, 502, 530, 506, 496, 528, 552, 518, 477, 472, 500, 496, 503, 513, 518, 542, 500, 477, 535, 528, 486, 533, 506, 518, 513, 519, 466, 543, 514, 476, 525, 502, 518, 492, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[516, 532, 511, 498, 500, 503, 524, 480, 500, 518, 500, 501, 508, 496, 515, 509, 533, 525, 506, 499, 554, 500, 480, 492, 511, 538, 549, 543, 513, 503, 538, 513, 535, 506, 484, 531, 547, 497, 541, 525, 495, 500, 507, 530, 490, 544, 539, 508, 527, 507, 509, 503, 503, 489, 493, 494, 494, 493, 504, 535, 511, 504, 516, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[527, 548, 512, 505, 490, 501, 496, 507, 501, 536, 539, 503, 524, 509, 525, 482, 526, 499, 506, 509, 502, 528, 497, 533, 527, 487, 484, 535, 525, 556, 486, 494, 512, 530, 491, 496, 523, 521, 456, 512, 525, 475, 511, 500, 487, 495, 554, 522, 533, 503, 528, 504, 490, 541, 537, 519, 487, 499, 556, 510, 526, 494, 511, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 484, 532, 481, 522, 522, 495, 515, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 526, 507, 543, 518, 478, 476, 0, 0, 0, 0, 0, 0, 0, 0, 531, 500, 539, 498, 496, 521, 491, 479, 19, 3, 0, 0, 0, 0, 0, 0, 489, 493, 517, 489, 540, 528, 520, 513, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 475, 503, 529, 477, 575, 558, 10, 4, 0, 0, 0, 0, 0, 0, 521, 498, 512, 494, 546, 522, 535, 472, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 483, 490, 493, 498, 517, 469, 31, 0, 0, 0, 0, 0, 0, 0, 538, 496, 524, 534, 530, 499, 514, 518, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4098, 4077, 4089, 4159, 4100, 4014, 4164] avg=4096.0000 max=4164 max_rank=7 min=4014 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 497, 531, 520, 495, 462, 518, 532, 0, 0, 0, 0, 0, 0, 0, 0, 478, 490, 521, 565, 509, 496, 519, 544, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 485, 528, 507, 511, 523, 489, 20, 3, 0, 0, 0, 0, 0, 0, 524, 525, 544, 499, 520, 511, 540, 455, 0, 0, 0, 0, 0, 0, 0, 0, 512, 489, 472, 509, 510, 527, 499, 511, 10, 3, 0, 0, 0, 0, 0, 0, 503, 521, 523, 504, 519, 497, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 505, 483, 528, 514, 516, 532, 522, 533, 32, 0, 0, 0, 0, 0, 0, 0, 501, 483, 496, 490, 511, 553, 513, 512, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4122, 4092, 4118, 4042, 4090, 4165, 4070] avg=4096.0000 max=4165 max_rank=6 min=4042 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[522, 466, 498, 515, 503, 508, 527, 537, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 510, 532, 511, 503, 487, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 529, 502, 510, 566, 522, 524, 500, 17, 3, 0, 0, 0, 0, 0, 0, 487, 507, 469, 533, 504, 495, 525, 525, 0, 0, 0, 0, 0, 0, 0, 0, 526, 526, 497, 517, 503, 517, 521, 513, 9, 3, 0, 0, 0, 0, 0, 0, 510, 525, 550, 542, 489, 507, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 486, 515, 502, 526, 497, 485, 538, 486, 30, 0, 0, 0, 0, 0, 0, 0, 520, 518, 479, 539, 517, 519, 478, 503, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4055, 4181, 4045, 4132, 4130, 4065, 4084] avg=4096.0000 max=4181 max_rank=2 min=4045 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 542, 501, 520, 519, 497, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 478, 519, 544, 503, 533, 473, 550, 527, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 478, 528, 493, 513, 479, 507, 18, 4, 0, 0, 0, 0, 0, 0, 520, 490, 495, 552, 496, 522, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 487, 487, 527, 504, 538, 478, 524, 502, 9, 4, 0, 0, 0, 0, 0, 0, 500, 526, 487, 536, 516, 507, 500, 493, 0, 0, 0, 0, 0, 0, 0, 0, 513, 469, 537, 508, 526, 539, 510, 531, 35, 0, 0, 0, 0, 0, 0, 0, 508, 509, 518, 509, 470, 489, 507, 536, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4127, 4070, 4085, 4060, 4065, 4168, 4057] avg=4096.0000 max=4168 max_rank=6 min=4057 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 517, 545, 534, 531, 548, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 519, 526, 500, 482, 540, 516, 0, 0, 0, 0, 0, 0, 0, 0, 495, 494, 505, 529, 492, 494, 532, 497, 18, 3, 0, 0, 0, 0, 0, 0, 509, 520, 486, 500, 506, 518, 544, 517, 0, 0, 0, 0, 0, 0, 0, 0, 519, 474, 525, 522, 467, 512, 533, 510, 9, 3, 0, 0, 0, 0, 0, 0, 476, 504, 502, 514, 538, 537, 525, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 499, 510, 471, 497, 466, 519, 505, 34, 0, 0, 0, 0, 0, 0, 0, 459, 489, 532, 535, 508, 510, 570, 499, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 4099, 4059, 4100, 4074, 4113, 4006, 4112] avg=4096.0000 max=4205 max_rank=0 min=4006 min_rank=6 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[527, 512, 512, 505, 490, 501, 496, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 539, 503, 510, 509, 525, 482, 0, 0, 0, 0, 0, 0, 0, 0, 526, 499, 506, 509, 502, 528, 497, 533, 17, 3, 0, 0, 0, 0, 0, 0, 527, 487, 458, 535, 525, 556, 486, 494, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 491, 496, 523, 521, 456, 512, 9, 3, 0, 0, 0, 0, 0, 0, 525, 475, 511, 500, 487, 495, 554, 522, 0, 0, 0, 0, 0, 0, 0, 0, 533, 503, 528, 504, 490, 541, 537, 519, 33, 0, 0, 0, 0, 0, 0, 0, 487, 499, 556, 510, 526, 494, 511, 521, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4105, 4120, 4068, 4053, 4069, 4188, 4115] avg=4096.0000 max=4188 max_rank=6 min=4050 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[549, 468, 540, 501, 510, 527, 548, 493, 0, 0, 0, 0, 0, 0, 0, 0, 496, 545, 512, 532, 478, 548, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 508, 501, 516, 492, 522, 497, 20, 8, 0, 0, 0, 0, 0, 0, 532, 527, 477, 509, 485, 522, 502, 530, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 528, 552, 518, 477, 472, 500, 12, 8, 0, 0, 0, 0, 0, 0, 496, 503, 513, 518, 542, 500, 477, 535, 4, 0, 0, 0, 0, 0, 0, 0, 528, 486, 533, 506, 518, 513, 519, 466, 36, 0, 0, 0, 0, 0, 0, 0, 543, 514, 476, 525, 502, 518, 492, 515, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4128, 4058, 4084, 4069, 4088, 4105, 4100] avg=4096.0000 max=4136 max_rank=0 min=4058 min_rank=2 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 497, 511, 498, 500, 503, 524, 480, 0, 0, 0, 0, 0, 0, 0, 0, 500, 518, 500, 501, 495, 496, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 525, 506, 499, 554, 500, 480, 492, 19, 3, 0, 0, 0, 0, 0, 0, 511, 538, 520, 543, 513, 503, 538, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 506, 484, 531, 547, 497, 541, 525, 10, 3, 0, 0, 0, 0, 0, 0, 495, 500, 507, 530, 490, 544, 539, 508, 0, 0, 0, 0, 0, 0, 0, 0, 527, 507, 509, 503, 503, 489, 493, 494, 32, 0, 0, 0, 0, 0, 0, 0, 494, 493, 504, 535, 511, 504, 516, 499, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4034, 4111, 4179, 4179, 4113, 4057, 4066] avg=4096.0000 max=4179 max_rank=3 min=4029 min_rank=0 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True + [2026-08-26 14:57:42] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 278.7 | throughput per GPU (TFLOP/s/GPU): 443.0 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.119382E+01 | loss scale: 1.0 | grad norm: 107.979 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[491, 529, 467, 488, 528, 522, 525, 486, 507, 511, 553, 470, 538, 531, 528, 535, 500, 499, 490, 535, 542, 496, 509, 513, 504, 507, 502, 512, 517, 531, 540, 492, 542, 512, 531, 507, 512, 529, 502, 541, 555, 549, 498, 491, 530, 513, 551, 511, 473, 528, 471, 523, 517, 509, 493, 502, 471, 521, 480, 496, 479, 496, 526, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[516, 572, 524, 532, 526, 505, 517, 488, 530, 522, 495, 498, 497, 541, 476, 500, 493, 502, 552, 533, 503, 529, 499, 529, 506, 519, 505, 482, 517, 512, 521, 498, 500, 528, 513, 498, 542, 505, 491, 495, 530, 486, 523, 517, 519, 525, 519, 498, 555, 490, 510, 512, 519, 494, 544, 489, 515, 474, 495, 509, 517, 497, 518, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[523, 528, 519, 495, 553, 526, 506, 526, 512, 512, 511, 505, 537, 523, 483, 537, 526, 515, 526, 499, 546, 522, 549, 472, 472, 528, 494, 502, 525, 513, 507, 544, 513, 530, 491, 436, 540, 523, 526, 549, 523, 522, 466, 523, 500, 497, 536, 495, 505, 510, 478, 508, 496, 512, 454, 527, 509, 496, 494, 511, 500, 537, 523, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[511, 529, 529, 501, 507, 497, 506, 554, 545, 522, 507, 476, 513, 507, 523, 541, 555, 510, 515, 533, 496, 507, 512, 511, 542, 516, 515, 467, 495, 507, 508, 505, 515, 505, 522, 452, 486, 547, 554, 504, 526, 486, 522, 496, 495, 497, 546, 508, 526, 545, 519, 509, 529, 491, 480, 504, 504, 491, 500, 485, 536, 522, 490, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[510, 510, 525, 496, 515, 507, 520, 504, 506, 504, 519, 496, 529, 519, 504, 508, 498, 547, 509, 483, 499, 505, 526, 529, 529, 532, 523, 512, 487, 501, 550, 508, 525, 495, 527, 513, 515, 480, 544, 503, 541, 558, 525, 463, 496, 522, 518, 464, 504, 500, 542, 526, 536, 530, 493, 479, 504, 553, 466, 515, 491, 468, 537, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[549, 480, 537, 489, 527, 523, 487, 528, 510, 557, 495, 519, 516, 493, 491, 513, 512, 536, 511, 495, 493, 491, 531, 509, 491, 480, 502, 506, 516, 529, 472, 547, 513, 529, 466, 492, 498, 522, 557, 515, 501, 512, 508, 534, 524, 520, 535, 511, 517, 533, 509, 537, 497, 491, 496, 495, 539, 520, 507, 533, 541, 496, 487, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[535, 463, 504, 504, 519, 500, 507, 535, 478, 497, 544, 549, 515, 519, 486, 518, 525, 508, 537, 505, 516, 515, 473, 515, 499, 529, 496, 522, 520, 509, 517, 548, 522, 544, 515, 520, 507, 509, 526, 500, 511, 509, 490, 509, 522, 507, 485, 503, 535, 493, 532, 522, 501, 504, 485, 504, 509, 510, 531, 508, 511, 533, 499, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[522, 491, 485, 520, 546, 513, 471, 494, 507, 540, 513, 515, 497, 558, 557, 525, 495, 509, 534, 546, 496, 528, 497, 524, 483, 540, 471, 547, 511, 539, 507, 537, 518, 521, 529, 517, 504, 476, 527, 499, 511, 523, 527, 509, 505, 474, 526, 493, 490, 499, 480, 537, 510, 512, 462, 522, 541, 490, 528, 524, 533, 518, 476, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 529, 467, 488, 519, 522, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 507, 511, 553, 470, 538, 504, 528, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 474, 535, 542, 496, 509, 513, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 502, 512, 517, 531, 540, 492, 0, 0, 0, 0, 0, 0, 0, 0, 542, 512, 531, 507, 512, 529, 491, 541, 0, 0, 0, 0, 0, 0, 0, 0, 555, 549, 498, 491, 530, 513, 549, 511, 0, 0, 0, 0, 0, 0, 0, 0, 473, 528, 471, 523, 517, 509, 493, 502, 11, 9, 9, 2, 0, 0, 0, 0, 471, 521, 480, 496, 479, 496, 526, 511, 27, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4146, 4068, 4105, 4165, 4196, 4047, 4014] avg=4096.0000 max=4196 max_rank=5 min=4014 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 510, 525, 496, 506, 507, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 506, 504, 519, 496, 529, 493, 504, 508, 0, 0, 0, 0, 0, 0, 0, 0, 498, 547, 492, 483, 499, 505, 526, 529, 0, 0, 0, 0, 0, 0, 0, 0, 529, 532, 523, 512, 487, 501, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0, 525, 495, 527, 513, 515, 480, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 541, 558, 525, 463, 496, 522, 516, 464, 0, 0, 0, 0, 0, 0, 0, 0, 504, 500, 542, 526, 536, 530, 493, 479, 12, 9, 9, 2, 0, 0, 0, 0, 504, 553, 466, 515, 491, 468, 537, 525, 26, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4059, 4079, 4142, 4090, 4085, 4142, 4093] avg=4096.0000 max=4142 max_rank=3 min=4059 min_rank=1 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[535, 463, 504, 504, 510, 500, 507, 535, 0, 0, 0, 0, 0, 0, 0, 0, 478, 497, 544, 549, 515, 493, 486, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 508, 519, 505, 516, 515, 473, 515, 0, 0, 0, 0, 0, 0, 0, 0, 499, 529, 496, 522, 520, 509, 517, 548, 0, 0, 0, 0, 0, 0, 0, 0, 522, 544, 515, 520, 507, 509, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 490, 509, 522, 507, 483, 503, 0, 0, 0, 0, 0, 0, 0, 0, 535, 493, 532, 522, 501, 504, 485, 504, 12, 10, 9, 2, 0, 0, 0, 0, 509, 510, 531, 508, 511, 533, 499, 505, 26, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4080, 4076, 4140, 4131, 4034, 4109, 4140] avg=4096.0000 max=4140 max_rank=3 min=4034 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 491, 485, 520, 537, 513, 471, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 540, 513, 515, 497, 530, 557, 525, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 516, 546, 496, 528, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 483, 540, 471, 547, 511, 539, 507, 537, 0, 0, 0, 0, 0, 0, 0, 0, 518, 521, 529, 517, 504, 476, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 523, 527, 509, 505, 474, 524, 493, 0, 0, 0, 0, 0, 0, 0, 0, 490, 499, 480, 537, 510, 512, 462, 522, 12, 10, 9, 2, 0, 0, 0, 0, 541, 490, 528, 524, 533, 518, 476, 469, 28, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4033, 4184, 4111, 4135, 4079, 4066, 4045, 4115] avg=4096.0000 max=4184 max_rank=1 min=4033 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 572, 524, 532, 517, 505, 517, 488, 0, 0, 0, 0, 0, 0, 0, 0, 530, 522, 495, 498, 497, 514, 476, 500, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 534, 533, 503, 529, 499, 529, 0, 0, 0, 0, 0, 0, 0, 0, 506, 519, 505, 482, 517, 512, 521, 498, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 513, 498, 542, 505, 480, 495, 0, 0, 0, 0, 0, 0, 0, 0, 530, 486, 523, 517, 519, 525, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 555, 490, 510, 512, 519, 494, 544, 489, 11, 10, 9, 2, 0, 0, 0, 0, 515, 474, 495, 509, 517, 497, 518, 502, 27, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4032, 4122, 4060, 4061, 4115, 4145, 4062] avg=4096.0000 max=4171 max_rank=0 min=4032 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[511, 529, 529, 501, 499, 497, 506, 554, 0, 0, 0, 0, 0, 0, 0, 0, 545, 522, 507, 476, 513, 481, 523, 541, 0, 0, 0, 0, 0, 0, 0, 0, 555, 510, 498, 533, 496, 507, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 542, 516, 515, 467, 495, 507, 508, 505, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 522, 452, 486, 547, 541, 504, 0, 0, 0, 0, 0, 0, 0, 0, 526, 486, 522, 496, 495, 497, 544, 508, 0, 0, 0, 0, 0, 0, 0, 0, 526, 545, 519, 509, 529, 491, 480, 504, 13, 9, 8, 2, 0, 0, 0, 0, 504, 491, 500, 485, 536, 522, 490, 514, 26, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4108, 4122, 4055, 4072, 4074, 4135, 4076] avg=4096.0000 max=4135 max_rank=6 min=4055 min_rank=3 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[523, 528, 519, 495, 544, 526, 506, 526, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 511, 505, 537, 497, 483, 537, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 508, 499, 546, 522, 549, 472, 0, 0, 0, 0, 0, 0, 0, 0, 472, 528, 494, 502, 525, 513, 507, 544, 0, 0, 0, 0, 0, 0, 0, 0, 513, 530, 491, 436, 540, 523, 514, 549, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 466, 523, 500, 497, 534, 495, 0, 0, 0, 0, 0, 0, 0, 0, 505, 510, 478, 508, 496, 512, 454, 527, 12, 10, 9, 2, 0, 0, 0, 0, 509, 496, 494, 511, 500, 537, 523, 502, 26, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4094, 4137, 4085, 4096, 4060, 4023, 4106] avg=4096.0000 max=4167 max_rank=0 min=4023 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 480, 537, 489, 515, 523, 487, 528, 0, 0, 0, 0, 0, 0, 0, 0, 510, 557, 495, 519, 516, 464, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 486, 495, 493, 491, 531, 509, 0, 0, 0, 0, 0, 0, 0, 0, 491, 480, 502, 506, 516, 529, 472, 547, 3, 0, 0, 0, 0, 0, 0, 0, 513, 529, 466, 492, 498, 522, 539, 515, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 508, 534, 524, 520, 528, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 509, 537, 497, 491, 496, 495, 18, 13, 12, 4, 0, 0, 0, 0, 539, 520, 507, 533, 541, 496, 487, 498, 29, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4065, 4053, 4046, 4074, 4138, 4122, 4162] avg=4096.0000 max=4162 max_rank=7 min=4046 min_rank=3 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 3968, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[520, 546, 509, 532, 550, 505, 504, 521, 479, 503, 550, 547, 498, 524, 505, 494, 522, 477, 474, 474, 503, 474, 520, 537, 512, 511, 521, 486, 531, 514, 482, 502, 505, 459, 448, 521, 499, 514, 521, 517, 514, 562, 516, 500, 521, 484, 547, 497, 538, 503, 499, 547, 476, 539, 506, 497, 545, 551, 496, 532, 553, 493, 511, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[490, 558, 508, 481, 513, 494, 508, 489, 553, 530, 522, 533, 499, 483, 493, 515, 496, 500, 527, 469, 509, 538, 518, 508, 524, 528, 532, 533, 519, 508, 530, 524, 542, 519, 544, 481, 501, 504, 501, 504, 504, 545, 471, 510, 477, 492, 516, 515, 491, 530, 516, 536, 506, 489, 501, 491, 526, 546, 538, 513, 493, 495, 533, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[536, 499, 528, 518, 544, 487, 508, 522, 509, 564, 473, 487, 544, 496, 467, 496, 529, 484, 496, 495, 531, 549, 517, 461, 522, 497, 548, 525, 476, 559, 505, 546, 501, 528, 517, 508, 507, 500, 515, 517, 510, 503, 511, 520, 513, 500, 521, 522, 512, 514, 529, 494, 493, 500, 493, 518, 491, 497, 517, 485, 509, 548, 510, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[510, 516, 513, 506, 530, 544, 532, 504, 503, 523, 486, 487, 512, 499, 510, 489, 498, 491, 508, 510, 564, 510, 490, 488, 534, 521, 521, 505, 519, 528, 532, 461, 510, 521, 496, 523, 479, 517, 518, 515, 477, 517, 550, 519, 528, 485, 532, 532, 527, 475, 511, 518, 524, 517, 505, 500, 515, 535, 512, 529, 487, 507, 510, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[489, 501, 537, 500, 507, 520, 532, 513, 534, 501, 515, 520, 534, 559, 485, 537, 534, 516, 499, 541, 507, 551, 511, 505, 513, 502, 554, 519, 497, 532, 508, 517, 477, 518, 531, 499, 549, 530, 511, 505, 507, 488, 508, 497, 486, 476, 526, 497, 535, 504, 501, 509, 498, 495, 524, 496, 493, 517, 499, 511, 492, 526, 501, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[486, 516, 507, 531, 499, 510, 488, 515, 562, 521, 533, 532, 518, 512, 501, 492, 507, 511, 549, 540, 515, 508, 501, 489, 502, 497, 534, 528, 523, 479, 472, 543, 530, 502, 479, 506, 500, 506, 528, 507, 510, 483, 536, 539, 540, 524, 496, 461, 497, 528, 510, 504, 533, 553, 504, 489, 549, 464, 520, 524, 505, 505, 512, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[500, 537, 530, 538, 515, 522, 476, 513, 498, 494, 526, 515, 528, 522, 532, 555, 485, 517, 517, 542, 503, 514, 524, 489, 541, 511, 498, 533, 526, 514, 517, 477, 488, 487, 488, 522, 530, 524, 528, 534, 502, 566, 467, 482, 485, 499, 478, 496, 542, 505, 491, 508, 522, 545, 516, 531, 503, 540, 502, 518, 487, 490, 504, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[505, 519, 533, 522, 493, 509, 496, 502, 510, 521, 496, 543, 521, 496, 521, 503, 502, 501, 515, 469, 521, 519, 546, 532, 531, 526, 516, 510, 494, 511, 547, 498, 507, 561, 528, 494, 531, 507, 502, 487, 525, 506, 502, 534, 551, 508, 534, 505, 530, 487, 495, 476, 471, 511, 519, 479, 504, 505, 467, 523, 549, 522, 511, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 486, 528, 518, 544, 487, 508, 522, 0, 0, 0, 0, 0, 0, 0, 0, 509, 564, 473, 472, 544, 496, 467, 496, 0, 0, 0, 0, 0, 0, 0, 0, 529, 484, 496, 495, 531, 549, 517, 461, 10, 3, 0, 0, 0, 0, 0, 0, 522, 497, 515, 525, 476, 559, 505, 546, 0, 0, 0, 0, 0, 0, 0, 0, 501, 528, 517, 508, 507, 500, 515, 517, 10, 3, 0, 0, 0, 0, 0, 0, 510, 503, 511, 520, 513, 500, 521, 522, 12, 5, 0, 0, 0, 0, 0, 0, 512, 514, 529, 494, 493, 500, 493, 518, 21, 0, 0, 0, 0, 0, 0, 0, 491, 494, 517, 485, 509, 548, 510, 547, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4021, 4075, 4145, 4106, 4117, 4074, 4101] avg=4096.0000 max=4145 max_rank=3 min=4021 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 532, 509, 532, 550, 505, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 479, 503, 550, 529, 498, 524, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 477, 474, 474, 503, 474, 520, 537, 12, 3, 0, 0, 0, 0, 0, 0, 512, 511, 490, 486, 531, 514, 482, 502, 0, 0, 0, 0, 0, 0, 0, 0, 505, 459, 448, 521, 499, 514, 521, 517, 11, 4, 0, 0, 0, 0, 0, 0, 514, 562, 516, 500, 521, 484, 547, 497, 11, 6, 0, 0, 0, 0, 0, 0, 538, 503, 499, 547, 476, 539, 506, 497, 20, 0, 0, 0, 0, 0, 0, 0, 545, 547, 496, 532, 553, 493, 511, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4082, 3996, 4028, 3999, 4158, 4125, 4207] avg=4096.0000 max=4207 max_rank=7 min=3996 min_rank=2 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 503, 513, 506, 530, 544, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 486, 472, 512, 499, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 508, 510, 564, 510, 490, 488, 10, 3, 0, 0, 0, 0, 0, 0, 534, 521, 490, 505, 519, 528, 532, 461, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 496, 523, 479, 517, 518, 515, 10, 3, 0, 0, 0, 0, 0, 0, 477, 517, 550, 519, 528, 485, 532, 532, 11, 5, 0, 0, 0, 0, 0, 0, 527, 475, 511, 518, 524, 517, 505, 500, 20, 0, 0, 0, 0, 0, 0, 0, 515, 532, 512, 529, 487, 507, 510, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 3994, 4072, 4090, 4092, 4156, 4097, 4125] avg=4096.0000 max=4156 max_rank=5 min=3994 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16470 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[490, 544, 508, 481, 513, 494, 508, 489, 0, 0, 0, 0, 0, 0, 0, 0, 553, 530, 522, 516, 499, 483, 493, 515, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 527, 469, 509, 538, 518, 508, 11, 3, 0, 0, 0, 0, 0, 0, 524, 528, 501, 533, 519, 508, 530, 524, 0, 0, 0, 0, 0, 0, 0, 0, 542, 519, 544, 481, 501, 504, 501, 504, 11, 4, 0, 0, 0, 0, 0, 0, 504, 545, 471, 510, 477, 492, 516, 515, 11, 6, 0, 0, 0, 0, 0, 0, 491, 530, 516, 536, 506, 489, 501, 491, 20, 0, 0, 0, 0, 0, 0, 0, 526, 542, 538, 513, 493, 495, 533, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4111, 4079, 4167, 4111, 4047, 4080, 4146] avg=4096.0000 max=4167 max_rank=3 min=4027 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[489, 488, 537, 500, 507, 520, 532, 513, 0, 0, 0, 0, 0, 0, 0, 0, 534, 501, 515, 503, 534, 559, 485, 537, 0, 0, 0, 0, 0, 0, 0, 0, 534, 516, 499, 541, 507, 551, 511, 505, 11, 3, 0, 0, 0, 0, 0, 0, 513, 502, 521, 519, 497, 532, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 477, 518, 531, 499, 549, 530, 511, 505, 10, 3, 0, 0, 0, 0, 0, 0, 507, 488, 508, 497, 486, 476, 526, 497, 12, 6, 0, 0, 0, 0, 0, 0, 535, 504, 501, 509, 498, 495, 524, 496, 21, 0, 0, 0, 0, 0, 0, 0, 493, 514, 499, 511, 492, 526, 501, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4168, 4178, 4109, 4133, 4003, 4083, 4008] avg=4096.0000 max=4178 max_rank=2 min=4003 min_rank=5 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16227 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[505, 505, 533, 522, 493, 509, 496, 502, 0, 0, 0, 0, 0, 0, 0, 0, 510, 521, 496, 526, 521, 496, 521, 503, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 515, 469, 521, 519, 546, 532, 11, 3, 0, 0, 0, 0, 0, 0, 531, 526, 485, 510, 494, 511, 547, 498, 0, 0, 0, 0, 0, 0, 0, 0, 507, 561, 528, 494, 531, 507, 502, 487, 11, 3, 0, 0, 0, 0, 0, 0, 525, 506, 502, 534, 551, 508, 534, 505, 11, 6, 0, 0, 0, 0, 0, 0, 530, 487, 495, 476, 471, 511, 519, 479, 20, 0, 0, 0, 0, 0, 0, 0, 504, 502, 467, 523, 549, 522, 511, 509, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4094, 4119, 4102, 4131, 4182, 3988, 4087] avg=4096.0000 max=4182 max_rank=5 min=3988 min_rank=6 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 493, 507, 531, 499, 510, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0, 562, 521, 533, 506, 518, 512, 501, 492, 0, 0, 0, 0, 0, 0, 0, 0, 507, 511, 549, 540, 515, 508, 501, 489, 16, 8, 0, 0, 0, 0, 0, 0, 502, 497, 495, 528, 523, 479, 472, 543, 0, 0, 0, 0, 0, 0, 0, 0, 530, 502, 479, 506, 500, 506, 528, 507, 15, 7, 0, 0, 0, 0, 0, 0, 510, 483, 536, 539, 540, 524, 496, 461, 15, 10, 0, 0, 0, 0, 0, 0, 497, 528, 510, 504, 533, 553, 504, 489, 24, 0, 0, 0, 0, 0, 0, 0, 549, 457, 520, 524, 505, 505, 512, 503, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4029, 4145, 4144, 4039, 4080, 4114, 4142, 4075] avg=4096.0000 max=4145 max_rank=1 min=4029 min_rank=0 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 523, 530, 538, 515, 522, 476, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 494, 526, 498, 528, 522, 532, 555, 0, 0, 0, 0, 0, 0, 0, 0, 485, 517, 517, 542, 503, 514, 524, 489, 11, 3, 0, 0, 0, 0, 0, 0, 541, 511, 469, 533, 526, 514, 517, 477, 0, 0, 0, 0, 0, 0, 0, 0, 488, 487, 488, 522, 530, 524, 528, 534, 11, 4, 0, 0, 0, 0, 0, 0, 502, 566, 467, 482, 485, 499, 478, 496, 10, 6, 0, 0, 0, 0, 0, 0, 542, 505, 491, 508, 522, 545, 516, 531, 19, 0, 0, 0, 0, 0, 0, 0, 503, 536, 502, 518, 487, 490, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4153, 4105, 4088, 4116, 3991, 4179, 4019] avg=4096.0000 max=4179 max_rank=6 min=3991 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[534, 556, 506, 517, 493, 501, 507, 526, 532, 542, 497, 551, 535, 507, 513, 498, 495, 510, 529, 527, 506, 547, 521, 498, 508, 479, 571, 492, 503, 541, 492, 488, 512, 493, 518, 532, 546, 482, 537, 558, 486, 504, 493, 549, 466, 503, 490, 515, 494, 543, 502, 503, 528, 478, 538, 451, 530, 489, 518, 467, 483, 534, 493, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[492, 498, 527, 439, 508, 508, 484, 517, 516, 508, 495, 489, 516, 507, 542, 534, 516, 522, 516, 525, 503, 501, 486, 535, 529, 541, 523, 528, 527, 487, 530, 449, 543, 507, 525, 508, 531, 480, 524, 504, 491, 493, 536, 516, 478, 520, 515, 503, 528, 539, 537, 511, 531, 496, 503, 548, 538, 484, 493, 504, 519, 520, 515, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[538, 493, 517, 492, 536, 529, 493, 475, 505, 551, 506, 518, 519, 483, 517, 512, 483, 523, 512, 528, 492, 538, 516, 496, 512, 483, 533, 541, 552, 531, 538, 462, 521, 526, 494, 468, 532, 521, 533, 528, 530, 545, 507, 538, 508, 482, 487, 523, 496, 512, 506, 480, 555, 528, 515, 465, 517, 513, 500, 507, 515, 520, 487, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[477, 508, 519, 549, 544, 513, 507, 512, 499, 554, 540, 526, 510, 510, 536, 525, 481, 480, 508, 506, 517, 510, 557, 524, 496, 506, 485, 521, 512, 502, 529, 507, 514, 459, 508, 527, 492, 557, 503, 510, 509, 528, 532, 523, 470, 520, 532, 518, 483, 510, 536, 523, 497, 506, 497, 505, 529, 502, 490, 527, 517, 492, 489, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[497, 524, 530, 507, 521, 485, 501, 544, 480, 544, 571, 520, 520, 547, 512, 499, 489, 487, 537, 535, 523, 523, 509, 535, 530, 513, 503, 494, 486, 509, 530, 510, 473, 509, 532, 496, 510, 495, 540, 507, 508, 487, 511, 480, 530, 514, 481, 530, 504, 518, 521, 518, 504, 490, 522, 505, 510, 527, 517, 478, 506, 518, 487, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[517, 526, 555, 549, 546, 536, 492, 497, 509, 462, 501, 525, 529, 510, 558, 531, 507, 558, 508, 499, 489, 522, 477, 520, 507, 482, 520, 544, 498, 502, 525, 510, 501, 523, 497, 496, 543, 494, 501, 514, 572, 513, 542, 528, 502, 497, 504, 512, 484, 539, 513, 509, 529, 490, 521, 469, 505, 500, 505, 520, 470, 475, 503, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[488, 522, 507, 498, 537, 473, 486, 524, 471, 556, 487, 528, 570, 522, 500, 525, 551, 491, 495, 520, 512, 485, 486, 476, 474, 549, 509, 510, 549, 539, 507, 523, 536, 514, 504, 526, 520, 534, 538, 505, 498, 514, 493, 494, 499, 523, 525, 512, 497, 488, 524, 545, 545, 507, 535, 515, 510, 493, 503, 494, 535, 487, 501, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[528, 498, 541, 547, 511, 516, 538, 479, 478, 536, 468, 511, 515, 543, 527, 516, 500, 494, 504, 517, 492, 520, 504, 517, 541, 527, 465, 512, 523, 506, 535, 503, 535, 526, 498, 520, 507, 519, 541, 509, 507, 496, 475, 520, 509, 547, 495, 478, 470, 483, 549, 499, 489, 507, 514, 514, 487, 529, 487, 541, 496, 572, 520, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 556, 499, 517, 493, 501, 507, 526, 0, 0, 0, 0, 0, 0, 0, 0, 532, 492, 497, 551, 535, 507, 513, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 510, 529, 527, 506, 547, 521, 498, 7, 1, 0, 0, 0, 0, 0, 0, 508, 479, 571, 492, 503, 541, 485, 488, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 518, 532, 546, 482, 514, 558, 0, 0, 0, 0, 0, 0, 0, 0, 486, 504, 493, 549, 466, 503, 490, 515, 17, 3, 0, 0, 0, 0, 0, 0, 494, 543, 502, 503, 528, 478, 538, 451, 6, 6, 0, 0, 0, 0, 0, 0, 530, 489, 518, 467, 483, 534, 493, 511, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4125, 4141, 4067, 4155, 4026, 4049, 4072] avg=4096.0000 max=4155 max_rank=4 min=4026 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16302 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[477, 508, 512, 549, 544, 513, 507, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 540, 526, 510, 510, 536, 525, 0, 0, 0, 0, 0, 0, 0, 0, 481, 480, 508, 506, 517, 510, 557, 524, 8, 1, 0, 0, 0, 0, 0, 0, 496, 506, 485, 521, 512, 502, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 514, 459, 508, 527, 492, 557, 481, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 528, 532, 523, 470, 520, 532, 518, 16, 3, 0, 0, 0, 0, 0, 0, 483, 510, 536, 523, 497, 506, 497, 505, 6, 6, 0, 0, 0, 0, 0, 0, 529, 502, 490, 527, 517, 492, 489, 493, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4149, 4092, 4050, 4048, 4151, 4069, 4087] avg=4096.0000 max=4151 max_rank=5 min=4048 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[492, 498, 520, 439, 508, 508, 484, 517, 0, 0, 0, 0, 0, 0, 0, 0, 516, 461, 495, 489, 516, 507, 542, 534, 0, 0, 0, 0, 0, 0, 0, 0, 516, 522, 516, 525, 503, 501, 486, 535, 8, 1, 0, 0, 0, 0, 0, 0, 529, 541, 523, 528, 527, 487, 522, 449, 0, 0, 0, 0, 0, 0, 0, 0, 543, 507, 525, 508, 531, 480, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 491, 493, 536, 516, 478, 520, 515, 503, 16, 3, 0, 0, 0, 0, 0, 0, 528, 539, 537, 511, 531, 496, 503, 548, 6, 6, 0, 0, 0, 0, 0, 0, 538, 484, 493, 504, 519, 520, 515, 530, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4060, 4113, 4106, 4100, 4071, 4205, 4147] avg=4096.0000 max=4205 max_rank=6 min=3966 min_rank=0 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16523 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[538, 493, 510, 492, 536, 529, 493, 475, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 506, 518, 519, 483, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 483, 523, 512, 528, 492, 538, 516, 496, 8, 1, 0, 0, 0, 0, 0, 0, 512, 483, 533, 541, 552, 531, 530, 462, 0, 0, 0, 0, 0, 0, 0, 0, 521, 526, 494, 468, 532, 521, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 545, 507, 538, 508, 482, 487, 523, 17, 3, 0, 0, 0, 0, 0, 0, 496, 512, 506, 480, 555, 528, 515, 465, 6, 6, 0, 0, 0, 0, 0, 0, 517, 513, 500, 507, 515, 520, 487, 485, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4061, 4097, 4144, 4100, 4140, 4069, 4091] avg=4096.0000 max=4144 max_rank=3 min=4061 min_rank=1 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16400 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[488, 522, 500, 498, 537, 473, 486, 524, 0, 0, 0, 0, 0, 0, 0, 0, 471, 505, 487, 528, 570, 522, 500, 525, 0, 0, 0, 0, 0, 0, 0, 0, 551, 491, 495, 520, 512, 485, 486, 476, 7, 1, 0, 0, 0, 0, 0, 0, 474, 549, 509, 510, 549, 539, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 504, 526, 520, 534, 515, 505, 0, 0, 0, 0, 0, 0, 0, 0, 498, 514, 493, 494, 499, 523, 525, 512, 17, 3, 0, 0, 0, 0, 0, 0, 497, 488, 524, 545, 545, 507, 535, 515, 6, 6, 0, 0, 0, 0, 0, 0, 510, 493, 503, 494, 535, 487, 501, 484, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4108, 4024, 4153, 4154, 4078, 4168, 4055] avg=4096.0000 max=4168 max_rank=6 min=4024 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 524, 523, 507, 521, 485, 501, 544, 0, 0, 0, 0, 0, 0, 0, 0, 480, 494, 571, 520, 520, 547, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 489, 487, 537, 535, 523, 523, 509, 535, 8, 1, 0, 0, 0, 0, 0, 0, 530, 513, 503, 494, 486, 509, 522, 510, 0, 0, 0, 0, 0, 0, 0, 0, 473, 509, 532, 496, 510, 495, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 508, 487, 511, 480, 530, 514, 481, 530, 17, 3, 0, 0, 0, 0, 0, 0, 504, 518, 521, 518, 504, 490, 522, 505, 6, 6, 0, 0, 0, 0, 0, 0, 510, 527, 517, 478, 506, 518, 487, 525, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4143, 4147, 4067, 4039, 4061, 4094, 4115] avg=4096.0000 max=4147 max_rank=2 min=4039 min_rank=4 imbalance_ratio=1.012451 max_load_reduction=28621 max_load_reduction_pct=0.873444 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[517, 526, 537, 549, 546, 536, 492, 497, 0, 0, 0, 0, 0, 0, 0, 0, 509, 414, 501, 525, 529, 510, 558, 531, 0, 0, 0, 0, 0, 0, 0, 0, 507, 558, 508, 499, 489, 522, 477, 520, 11, 6, 0, 0, 0, 0, 0, 0, 507, 482, 520, 544, 498, 502, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 501, 523, 497, 496, 543, 494, 472, 514, 0, 0, 0, 0, 0, 0, 0, 0, 572, 513, 542, 528, 502, 497, 504, 512, 18, 6, 0, 0, 0, 0, 0, 0, 484, 539, 513, 509, 529, 490, 521, 469, 12, 11, 0, 0, 0, 0, 0, 0, 505, 500, 505, 520, 470, 475, 503, 486, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4077, 4097, 4077, 4040, 4194, 4077, 4006] avg=4096.0000 max=4200 max_rank=0 min=4006 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16317 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[528, 498, 534, 547, 511, 516, 538, 479, 0, 0, 0, 0, 0, 0, 0, 0, 478, 487, 468, 511, 515, 543, 527, 516, 0, 0, 0, 0, 0, 0, 0, 0, 500, 494, 504, 517, 492, 520, 504, 517, 8, 1, 0, 0, 0, 0, 0, 0, 541, 527, 465, 512, 523, 506, 527, 503, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 498, 520, 507, 519, 518, 509, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 475, 520, 509, 547, 495, 478, 17, 3, 0, 0, 0, 0, 0, 0, 470, 483, 549, 499, 489, 507, 514, 514, 6, 6, 0, 0, 0, 0, 0, 0, 487, 529, 487, 541, 496, 572, 520, 517, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4045, 4057, 4104, 4132, 4047, 4037, 4195] avg=4096.0000 max=4195 max_rank=7 min=4037 min_rank=6 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[475, 520, 519, 514, 514, 549, 514, 485, 530, 504, 517, 528, 519, 472, 517, 492, 516, 507, 531, 520, 540, 499, 529, 563, 555, 496, 519, 508, 529, 485, 512, 467, 490, 502, 509, 487, 479, 517, 503, 540, 519, 516, 533, 516, 488, 538, 482, 503, 496, 517, 510, 509, 499, 520, 491, 511, 503, 515, 486, 525, 530, 530, 501, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[553, 528, 506, 484, 509, 495, 523, 504, 493, 502, 500, 488, 531, 510, 502, 481, 516, 479, 538, 511, 493, 526, 524, 554, 512, 531, 541, 494, 501, 521, 529, 506, 515, 528, 513, 521, 514, 520, 526, 543, 522, 523, 505, 486, 532, 541, 518, 504, 497, 507, 528, 492, 488, 520, 511, 489, 545, 508, 480, 485, 542, 518, 482, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[544, 529, 488, 542, 510, 552, 500, 529, 502, 534, 522, 527, 517, 507, 512, 515, 539, 516, 496, 518, 523, 501, 556, 475, 487, 492, 530, 480, 508, 491, 522, 544, 521, 524, 519, 519, 492, 532, 516, 507, 524, 493, 467, 517, 517, 517, 500, 491, 490, 495, 515, 485, 472, 559, 526, 506, 499, 481, 516, 526, 489, 503, 524, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[476, 543, 514, 496, 566, 532, 549, 494, 537, 529, 500, 498, 516, 518, 558, 482, 539, 535, 568, 495, 519, 511, 523, 537, 524, 484, 525, 518, 516, 499, 443, 468, 484, 498, 473, 501, 529, 539, 492, 514, 507, 525, 472, 495, 494, 517, 512, 497, 498, 530, 517, 539, 482, 495, 489, 486, 518, 517, 532, 545, 532, 491, 499, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[507, 494, 561, 505, 534, 522, 497, 518, 529, 483, 520, 546, 524, 515, 507, 530, 533, 501, 569, 506, 518, 506, 553, 540, 485, 511, 496, 498, 536, 478, 530, 512, 475, 534, 503, 499, 509, 528, 503, 504, 484, 484, 538, 512, 517, 535, 495, 480, 504, 546, 532, 495, 515, 531, 511, 497, 522, 486, 494, 465, 520, 488, 473, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[508, 538, 515, 530, 517, 517, 470, 529, 569, 531, 491, 510, 529, 532, 523, 521, 511, 508, 495, 522, 517, 489, 525, 561, 518, 509, 527, 499, 477, 489, 482, 536, 564, 494, 503, 518, 491, 525, 490, 496, 544, 483, 530, 480, 500, 502, 518, 506, 487, 479, 517, 515, 481, 533, 489, 505, 509, 514, 552, 534, 502, 519, 502, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[500, 506, 523, 526, 525, 507, 487, 552, 511, 542, 511, 544, 516, 509, 515, 524, 541, 527, 547, 517, 497, 498, 469, 481, 514, 484, 540, 513, 510, 500, 517, 486, 546, 528, 477, 506, 526, 526, 547, 518, 511, 520, 451, 530, 517, 498, 474, 505, 501, 499, 508, 509, 512, 510, 511, 476, 502, 522, 523, 494, 470, 549, 535, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[536, 503, 545, 532, 550, 525, 515, 490, 509, 534, 525, 479, 549, 549, 498, 498, 518, 547, 535, 537, 493, 464, 526, 468, 531, 487, 492, 532, 511, 523, 524, 537, 463, 520, 520, 481, 519, 540, 504, 494, 509, 507, 504, 497, 527, 544, 527, 488, 472, 525, 536, 473, 529, 490, 507, 491, 488, 475, 540, 513, 537, 527, 495, 464] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[475, 520, 519, 514, 470, 549, 514, 485, 0, 0, 0, 0, 0, 0, 0, 0, 530, 504, 517, 528, 488, 472, 517, 492, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 476, 520, 540, 499, 529, 563, 0, 0, 0, 0, 0, 0, 0, 0, 555, 496, 519, 508, 529, 485, 512, 467, 15, 14, 0, 0, 0, 0, 0, 0, 490, 502, 509, 487, 479, 517, 503, 540, 6, 0, 0, 0, 0, 0, 0, 0, 519, 516, 533, 516, 488, 538, 482, 503, 29, 4, 0, 0, 0, 0, 0, 0, 496, 517, 510, 509, 499, 520, 491, 511, 51, 0, 0, 0, 0, 0, 0, 0, 503, 515, 486, 525, 530, 530, 501, 558, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4048, 4150, 4100, 4033, 4128, 4104, 4159] avg=4096.0000 max=4159 max_rank=7 min=4033 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[553, 528, 506, 484, 465, 495, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 500, 488, 498, 510, 502, 481, 0, 0, 0, 0, 0, 0, 0, 0, 516, 479, 483, 511, 493, 526, 524, 554, 0, 0, 0, 0, 0, 0, 0, 0, 512, 531, 541, 494, 501, 521, 529, 506, 15, 15, 0, 0, 0, 0, 0, 0, 515, 528, 513, 521, 514, 520, 526, 543, 6, 0, 0, 0, 0, 0, 0, 0, 522, 523, 505, 486, 532, 541, 518, 504, 29, 4, 0, 0, 0, 0, 0, 0, 497, 507, 528, 492, 488, 520, 511, 489, 51, 0, 0, 0, 0, 0, 0, 0, 545, 508, 480, 485, 542, 518, 482, 480, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 3974, 4086, 4165, 4186, 4164, 4083, 4052] avg=4096.0000 max=4186 max_rank=4 min=3974 min_rank=1 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16485 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[476, 543, 514, 496, 517, 532, 549, 494, 0, 0, 0, 0, 0, 0, 0, 0, 537, 529, 500, 498, 485, 518, 558, 482, 0, 0, 0, 0, 0, 0, 0, 0, 539, 535, 510, 495, 519, 511, 523, 537, 0, 0, 0, 0, 0, 0, 0, 0, 524, 484, 525, 518, 516, 499, 443, 468, 17, 14, 0, 0, 0, 0, 0, 0, 484, 498, 473, 501, 529, 539, 492, 514, 6, 0, 0, 0, 0, 0, 0, 0, 507, 525, 472, 495, 494, 517, 512, 497, 32, 4, 0, 0, 0, 0, 0, 0, 498, 530, 517, 539, 482, 495, 489, 486, 54, 0, 0, 0, 0, 0, 0, 0, 518, 517, 532, 545, 532, 491, 499, 537, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4107, 4169, 4008, 4036, 4055, 4090, 4182] avg=4096.0000 max=4182 max_rank=7 min=4008 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 494, 561, 505, 488, 522, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 529, 483, 520, 546, 493, 515, 507, 530, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 511, 506, 518, 506, 553, 540, 0, 0, 0, 0, 0, 0, 0, 0, 485, 511, 496, 498, 536, 478, 530, 512, 16, 14, 0, 0, 0, 0, 0, 0, 475, 534, 503, 499, 509, 528, 503, 504, 6, 0, 0, 0, 0, 0, 0, 0, 484, 484, 538, 512, 517, 535, 495, 480, 30, 4, 0, 0, 0, 0, 0, 0, 504, 546, 532, 495, 515, 531, 511, 497, 54, 0, 0, 0, 0, 0, 0, 0, 522, 486, 494, 465, 520, 488, 473, 525, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4123, 4168, 4076, 4061, 4079, 4185, 3984] avg=4096.0000 max=4185 max_rank=6 min=3984 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[544, 529, 488, 542, 466, 552, 500, 529, 0, 0, 0, 0, 0, 0, 0, 0, 502, 534, 522, 527, 486, 507, 512, 515, 0, 0, 0, 0, 0, 0, 0, 0, 539, 516, 445, 518, 523, 501, 556, 475, 0, 0, 0, 0, 0, 0, 0, 0, 487, 492, 530, 480, 508, 491, 522, 544, 15, 14, 0, 0, 0, 0, 0, 0, 521, 524, 519, 519, 492, 532, 516, 507, 6, 0, 0, 0, 0, 0, 0, 0, 524, 493, 467, 517, 517, 517, 500, 491, 29, 4, 0, 0, 0, 0, 0, 0, 490, 495, 515, 485, 472, 559, 526, 506, 47, 0, 0, 0, 0, 0, 0, 0, 499, 481, 516, 526, 489, 503, 524, 518, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4105, 4073, 4083, 4136, 4059, 4095, 4067] avg=4096.0000 max=4150 max_rank=0 min=4059 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 506, 523, 526, 480, 507, 487, 552, 0, 0, 0, 0, 0, 0, 0, 0, 511, 542, 511, 544, 485, 509, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 541, 527, 491, 517, 497, 498, 469, 481, 0, 0, 0, 0, 0, 0, 0, 0, 514, 484, 540, 513, 510, 500, 517, 486, 15, 14, 0, 0, 0, 0, 0, 0, 546, 528, 477, 506, 526, 526, 547, 518, 6, 0, 0, 0, 0, 0, 0, 0, 511, 520, 451, 530, 517, 498, 474, 505, 30, 4, 0, 0, 0, 0, 0, 0, 501, 499, 508, 509, 512, 510, 511, 476, 52, 0, 0, 0, 0, 0, 0, 0, 502, 522, 523, 494, 470, 549, 535, 528, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4141, 4021, 4093, 4180, 4040, 4078, 4134] avg=4096.0000 max=4180 max_rank=4 min=4021 min_rank=2 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16432 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[508, 538, 515, 530, 464, 517, 470, 529, 0, 0, 0, 0, 0, 0, 0, 0, 569, 531, 491, 510, 485, 532, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 435, 522, 517, 489, 525, 561, 0, 0, 0, 0, 0, 0, 0, 0, 518, 509, 527, 499, 477, 489, 482, 536, 18, 20, 0, 0, 0, 0, 0, 0, 564, 494, 503, 518, 491, 525, 490, 496, 8, 0, 0, 0, 0, 0, 0, 0, 544, 483, 530, 480, 500, 502, 518, 506, 35, 7, 0, 0, 0, 0, 0, 0, 487, 479, 517, 515, 481, 533, 489, 505, 53, 0, 0, 0, 0, 0, 0, 0, 509, 514, 552, 534, 502, 519, 502, 491, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4162, 4068, 4075, 4089, 4105, 4059, 4139] avg=4096.0000 max=4162 max_rank=1 min=4059 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[536, 503, 545, 532, 503, 525, 515, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 534, 525, 479, 516, 549, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 518, 547, 480, 537, 493, 464, 526, 468, 0, 0, 0, 0, 0, 0, 0, 0, 531, 487, 492, 532, 511, 523, 524, 537, 16, 15, 0, 0, 0, 0, 0, 0, 463, 520, 520, 481, 519, 540, 504, 494, 6, 0, 0, 0, 0, 0, 0, 0, 509, 507, 504, 497, 527, 544, 527, 488, 31, 4, 0, 0, 0, 0, 0, 0, 472, 525, 536, 473, 529, 490, 507, 491, 51, 0, 0, 0, 0, 0, 0, 0, 488, 475, 540, 513, 537, 527, 495, 464, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4108, 4033, 4168, 4047, 4138, 4074, 4051] avg=4096.0000 max=4168 max_rank=3 min=4033 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4096, 3968, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4352, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[497, 504, 484, 505, 471, 501, 466, 525, 528, 523, 549, 479, 542, 527, 516, 509, 512, 537, 531, 527, 526, 484, 522, 505, 481, 543, 478, 493, 486, 518, 535, 520, 507, 495, 528, 509, 515, 482, 524, 485, 524, 521, 564, 528, 490, 521, 513, 519, 525, 526, 528, 484, 526, 572, 486, 479, 533, 539, 498, 518, 509, 516, 505, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[500, 484, 536, 520, 504, 515, 526, 540, 579, 500, 516, 514, 482, 518, 462, 521, 482, 510, 548, 514, 502, 522, 521, 526, 536, 486, 538, 505, 525, 518, 528, 525, 527, 531, 542, 545, 462, 527, 537, 465, 488, 514, 543, 526, 477, 513, 473, 446, 490, 513, 534, 493, 536, 526, 488, 499, 519, 498, 537, 513, 509, 508, 483, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[526, 504, 530, 481, 505, 467, 508, 510, 500, 530, 532, 517, 492, 532, 546, 503, 500, 509, 527, 508, 527, 517, 496, 531, 553, 546, 509, 497, 498, 547, 519, 526, 544, 456, 477, 506, 486, 505, 516, 482, 518, 505, 504, 519, 523, 530, 472, 518, 500, 489, 490, 526, 509, 490, 493, 552, 537, 518, 524, 489, 524, 559, 489, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[555, 530, 483, 492, 523, 496, 499, 547, 520, 510, 518, 514, 494, 526, 532, 502, 573, 495, 496, 505, 511, 489, 511, 540, 511, 524, 490, 472, 514, 531, 527, 512, 509, 549, 506, 466, 535, 528, 526, 476, 497, 527, 516, 526, 547, 518, 500, 513, 535, 523, 499, 476, 485, 482, 538, 532, 511, 514, 488, 468, 527, 488, 494, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[530, 506, 512, 533, 481, 510, 483, 485, 519, 501, 546, 494, 515, 510, 550, 476, 535, 543, 475, 497, 475, 506, 521, 494, 514, 521, 536, 503, 515, 465, 494, 522, 529, 538, 517, 514, 497, 507, 500, 529, 523, 536, 529, 509, 488, 524, 493, 534, 525, 472, 523, 520, 510, 514, 506, 545, 498, 511, 500, 496, 514, 526, 530, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[484, 495, 500, 543, 511, 495, 531, 529, 511, 487, 518, 518, 523, 497, 535, 487, 529, 548, 546, 522, 506, 517, 534, 501, 517, 519, 520, 500, 512, 536, 533, 513, 505, 511, 519, 516, 509, 519, 525, 520, 515, 517, 516, 534, 484, 524, 460, 490, 521, 513, 490, 503, 534, 499, 506, 482, 502, 517, 504, 438, 487, 526, 547, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[532, 507, 501, 537, 536, 524, 518, 511, 548, 507, 501, 510, 518, 519, 501, 475, 535, 486, 543, 495, 481, 501, 511, 503, 515, 510, 518, 550, 519, 480, 508, 507, 525, 506, 497, 531, 486, 511, 464, 492, 554, 500, 541, 509, 490, 527, 503, 500, 511, 499, 477, 495, 518, 532, 512, 513, 485, 495, 538, 533, 539, 545, 513, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[514, 504, 534, 536, 519, 535, 488, 525, 490, 505, 490, 537, 517, 485, 471, 512, 512, 515, 531, 524, 525, 543, 529, 505, 461, 507, 515, 547, 559, 507, 539, 500, 528, 528, 544, 544, 538, 506, 518, 511, 484, 512, 507, 470, 558, 456, 521, 483, 523, 483, 518, 519, 500, 493, 503, 506, 534, 474, 495, 491, 516, 478, 518, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 504, 484, 505, 471, 501, 466, 525, 5, 5, 0, 0, 0, 0, 0, 0, 524, 523, 549, 479, 542, 527, 516, 509, 0, 0, 0, 0, 0, 0, 0, 0, 512, 537, 504, 527, 526, 484, 522, 505, 0, 0, 0, 0, 0, 0, 0, 0, 481, 543, 478, 493, 486, 518, 513, 520, 0, 0, 0, 0, 0, 0, 0, 0, 507, 495, 528, 509, 515, 482, 524, 485, 4, 0, 0, 0, 0, 0, 0, 0, 524, 521, 564, 528, 490, 521, 513, 519, 10, 0, 0, 0, 0, 0, 0, 0, 525, 526, 528, 484, 526, 572, 486, 479, 22, 0, 0, 0, 0, 0, 0, 0, 533, 539, 498, 518, 509, 516, 505, 475, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3963, 4169, 4117, 4032, 4049, 4190, 4148, 4100] avg=4096.0000 max=4190 max_rank=5 min=3963 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[526, 504, 530, 481, 505, 467, 508, 510, 5, 4, 0, 0, 0, 0, 0, 0, 496, 530, 532, 517, 492, 532, 546, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 509, 500, 508, 527, 517, 496, 531, 0, 0, 0, 0, 0, 0, 0, 0, 553, 546, 509, 497, 498, 547, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 544, 456, 477, 506, 486, 505, 516, 482, 4, 0, 0, 0, 0, 0, 0, 0, 518, 505, 504, 519, 523, 530, 472, 518, 10, 0, 0, 0, 0, 0, 0, 0, 500, 489, 490, 526, 509, 490, 493, 552, 22, 0, 0, 0, 0, 0, 0, 0, 537, 518, 524, 489, 524, 559, 489, 525, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4148, 4088, 4174, 3976, 4099, 4071, 4172] avg=4096.0000 max=4174 max_rank=3 min=3976 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 484, 536, 520, 504, 515, 526, 540, 5, 5, 0, 0, 0, 0, 0, 0, 575, 500, 516, 514, 482, 518, 462, 521, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 520, 514, 502, 522, 521, 526, 0, 0, 0, 0, 0, 0, 0, 0, 536, 486, 538, 505, 525, 518, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 527, 531, 542, 545, 462, 527, 537, 465, 4, 0, 0, 0, 0, 0, 0, 0, 488, 514, 543, 526, 477, 513, 473, 446, 10, 0, 0, 0, 0, 0, 0, 0, 490, 513, 534, 493, 536, 526, 488, 499, 23, 0, 0, 0, 0, 0, 0, 0, 519, 498, 537, 513, 509, 508, 483, 503, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4088, 4097, 4139, 4140, 3990, 4102, 4077] avg=4096.0000 max=4140 max_rank=4 min=3990 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[555, 530, 483, 492, 523, 496, 499, 547, 5, 5, 0, 0, 0, 0, 0, 0, 516, 510, 518, 514, 494, 526, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 573, 495, 470, 505, 511, 489, 511, 540, 0, 0, 0, 0, 0, 0, 0, 0, 511, 524, 490, 472, 514, 531, 505, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 549, 506, 466, 535, 528, 526, 476, 4, 0, 0, 0, 0, 0, 0, 0, 497, 527, 516, 526, 547, 518, 500, 513, 10, 0, 0, 0, 0, 0, 0, 0, 535, 523, 499, 476, 485, 482, 538, 532, 21, 0, 0, 0, 0, 0, 0, 0, 511, 514, 488, 468, 527, 488, 494, 527, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4112, 4094, 4059, 4099, 4154, 4091, 4024] avg=4096.0000 max=4154 max_rank=5 min=4024 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 506, 512, 533, 481, 510, 483, 485, 5, 4, 0, 0, 0, 0, 0, 0, 515, 501, 546, 494, 515, 510, 550, 476, 0, 0, 0, 0, 0, 0, 0, 0, 535, 543, 450, 497, 475, 506, 521, 494, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 536, 503, 515, 465, 474, 522, 0, 0, 0, 0, 0, 0, 0, 0, 529, 538, 517, 514, 497, 507, 500, 529, 4, 0, 0, 0, 0, 0, 0, 0, 523, 536, 529, 509, 488, 524, 493, 534, 9, 0, 0, 0, 0, 0, 0, 0, 525, 472, 523, 520, 510, 514, 506, 545, 20, 0, 0, 0, 0, 0, 0, 0, 498, 511, 500, 496, 514, 526, 530, 544, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4107, 4021, 4050, 4135, 4145, 4135, 4126] avg=4096.0000 max=4145 max_rank=5 min=4021 min_rank=2 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16541 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 504, 534, 536, 519, 535, 488, 525, 5, 5, 0, 0, 0, 0, 0, 0, 486, 505, 490, 537, 517, 485, 471, 512, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 504, 524, 525, 543, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0, 461, 507, 515, 547, 559, 507, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 528, 528, 544, 544, 538, 506, 518, 511, 4, 0, 0, 0, 0, 0, 0, 0, 484, 512, 507, 470, 558, 456, 521, 483, 10, 0, 0, 0, 0, 0, 0, 0, 523, 483, 518, 519, 500, 493, 503, 506, 22, 0, 0, 0, 0, 0, 0, 0, 534, 474, 495, 491, 516, 478, 518, 528, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4003, 4157, 4113, 4221, 4001, 4067, 4041] avg=4096.0000 max=4221 max_rank=4 min=4001 min_rank=5 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[484, 495, 500, 543, 511, 495, 531, 529, 10, 8, 0, 0, 0, 0, 0, 0, 501, 487, 518, 518, 523, 497, 535, 487, 0, 0, 0, 0, 0, 0, 0, 0, 529, 548, 510, 522, 506, 517, 534, 501, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 520, 500, 512, 536, 498, 513, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 519, 516, 509, 519, 525, 520, 8, 0, 0, 0, 0, 0, 0, 0, 515, 517, 516, 534, 484, 524, 460, 490, 15, 0, 0, 0, 0, 0, 0, 0, 521, 513, 490, 503, 534, 499, 506, 482, 26, 0, 0, 0, 0, 0, 0, 0, 502, 517, 504, 438, 487, 526, 547, 518, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4066, 4167, 4115, 4132, 4055, 4074, 4053] avg=4096.0000 max=4167 max_rank=2 min=4053 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[532, 507, 501, 537, 536, 524, 518, 511, 5, 4, 0, 0, 0, 0, 0, 0, 544, 507, 501, 510, 518, 519, 501, 475, 0, 0, 0, 0, 0, 0, 0, 0, 535, 486, 515, 495, 481, 501, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 518, 550, 519, 480, 487, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 506, 497, 531, 486, 511, 464, 492, 4, 0, 0, 0, 0, 0, 0, 0, 554, 500, 541, 509, 490, 527, 503, 500, 10, 0, 0, 0, 0, 0, 0, 0, 511, 499, 477, 495, 518, 532, 512, 513, 23, 0, 0, 0, 0, 0, 0, 0, 485, 495, 538, 533, 539, 545, 513, 520, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4075, 4027, 4086, 4016, 4134, 4080, 4175] avg=4096.0000 max=4175 max_rank=0 min=4016 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 3968, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[521, 493, 519, 482, 538, 512, 497, 546, 540, 508, 498, 502, 508, 526, 543, 483, 534, 492, 500, 485, 478, 505, 573, 535, 523, 525, 515, 508, 511, 550, 492, 558, 510, 502, 480, 530, 525, 498, 511, 544, 501, 485, 482, 462, 520, 522, 490, 482, 492, 502, 497, 499, 531, 486, 555, 510, 507, 560, 505, 524, 523, 525, 490, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[513, 555, 535, 502, 522, 500, 528, 511, 534, 506, 487, 552, 524, 503, 517, 525, 540, 496, 499, 545, 542, 494, 484, 494, 502, 528, 469, 497, 497, 482, 488, 529, 496, 523, 514, 538, 530, 505, 509, 544, 498, 513, 537, 530, 527, 497, 499, 538, 529, 499, 512, 495, 503, 465, 519, 527, 475, 483, 480, 538, 510, 502, 514, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[543, 506, 518, 545, 522, 526, 519, 520, 514, 499, 523, 494, 453, 538, 528, 511, 503, 450, 532, 515, 501, 502, 518, 502, 519, 524, 514, 485, 535, 530, 489, 534, 514, 511, 519, 501, 483, 533, 568, 480, 514, 513, 539, 477, 499, 514, 483, 543, 511, 489, 493, 522, 511, 521, 538, 473, 523, 493, 512, 480, 559, 527, 505, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[531, 498, 508, 530, 503, 504, 532, 477, 508, 540, 516, 543, 502, 498, 512, 494, 509, 509, 518, 513, 496, 484, 518, 538, 505, 512, 477, 520, 547, 535, 552, 514, 521, 490, 526, 475, 550, 470, 541, 518, 521, 507, 542, 524, 488, 529, 480, 494, 524, 492, 502, 548, 489, 524, 500, 543, 482, 518, 513, 511, 498, 500, 497, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[496, 523, 529, 531, 500, 468, 493, 517, 531, 501, 528, 518, 482, 554, 540, 514, 502, 532, 494, 493, 509, 519, 494, 522, 477, 487, 518, 488, 534, 488, 527, 487, 524, 530, 520, 503, 511, 523, 485, 522, 521, 550, 536, 478, 518, 519, 483, 486, 502, 512, 490, 531, 536, 540, 518, 547, 517, 511, 543, 482, 518, 502, 537, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[493, 547, 567, 538, 523, 477, 551, 529, 534, 518, 509, 529, 525, 485, 486, 561, 494, 536, 519, 499, 525, 471, 519, 506, 496, 455, 483, 519, 517, 507, 494, 515, 518, 549, 514, 535, 488, 509, 508, 507, 519, 533, 518, 535, 505, 510, 497, 494, 504, 496, 512, 542, 497, 478, 511, 520, 535, 468, 492, 493, 503, 510, 525, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[481, 453, 535, 493, 518, 502, 500, 531, 530, 506, 544, 508, 512, 520, 517, 470, 488, 538, 522, 527, 507, 509, 465, 483, 510, 535, 516, 560, 534, 488, 477, 511, 550, 516, 495, 518, 503, 508, 563, 537, 537, 507, 489, 496, 504, 527, 535, 516, 522, 483, 550, 507, 489, 527, 536, 510, 529, 499, 509, 495, 495, 498, 538, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[496, 529, 496, 491, 512, 526, 517, 541, 484, 540, 526, 530, 502, 535, 494, 531, 484, 490, 528, 530, 521, 522, 531, 518, 494, 516, 501, 504, 531, 487, 517, 538, 528, 509, 508, 506, 508, 500, 498, 476, 538, 551, 514, 501, 491, 537, 502, 526, 449, 540, 510, 512, 464, 520, 492, 516, 514, 505, 501, 491, 549, 528, 495, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[521, 493, 497, 482, 538, 512, 497, 546, 0, 0, 0, 0, 0, 0, 0, 0, 540, 508, 498, 476, 508, 526, 543, 483, 0, 0, 0, 0, 0, 0, 0, 0, 534, 492, 500, 485, 478, 505, 573, 535, 13, 6, 0, 0, 0, 0, 0, 0, 523, 525, 515, 508, 511, 550, 492, 558, 11, 0, 0, 0, 0, 0, 0, 0, 510, 502, 480, 530, 525, 498, 493, 544, 0, 0, 0, 0, 0, 0, 0, 0, 501, 485, 482, 462, 520, 522, 490, 482, 5, 0, 0, 0, 0, 0, 0, 0, 492, 502, 497, 499, 531, 486, 555, 510, 9, 2, 0, 0, 0, 0, 0, 0, 507, 560, 505, 524, 523, 525, 490, 518, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4082, 4121, 4193, 4082, 3949, 4083, 4172] avg=4096.0000 max=4193 max_rank=3 min=3949 min_rank=5 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16286 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[543, 506, 496, 545, 522, 526, 519, 520, 0, 0, 0, 0, 0, 0, 0, 0, 514, 499, 523, 469, 453, 538, 528, 511, 0, 0, 0, 0, 0, 0, 0, 0, 503, 450, 532, 515, 501, 502, 518, 502, 13, 6, 0, 0, 0, 0, 0, 0, 519, 524, 514, 485, 535, 530, 489, 534, 12, 0, 0, 0, 0, 0, 0, 0, 514, 511, 519, 501, 483, 533, 547, 480, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 539, 477, 499, 514, 483, 543, 6, 0, 0, 0, 0, 0, 0, 0, 511, 489, 493, 522, 511, 521, 538, 473, 9, 3, 0, 0, 0, 0, 0, 0, 523, 493, 512, 480, 559, 527, 505, 508, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4035, 4042, 4142, 4088, 4088, 4070, 4126] avg=4096.0000 max=4177 max_rank=0 min=4035 min_rank=1 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 555, 511, 502, 522, 500, 528, 511, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 487, 523, 524, 503, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 540, 496, 499, 545, 542, 494, 484, 494, 14, 7, 0, 0, 0, 0, 0, 0, 502, 528, 469, 497, 497, 482, 488, 529, 11, 0, 0, 0, 0, 0, 0, 0, 496, 523, 514, 538, 530, 505, 491, 544, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 537, 530, 527, 497, 499, 538, 5, 0, 0, 0, 0, 0, 0, 0, 529, 499, 512, 495, 503, 465, 519, 527, 10, 2, 0, 0, 0, 0, 0, 0, 475, 483, 480, 538, 510, 502, 514, 519, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4119, 4115, 4003, 4141, 4144, 4061, 4043] avg=4096.0000 max=4144 max_rank=5 min=4003 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[531, 498, 486, 530, 503, 504, 532, 477, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 516, 515, 502, 498, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 518, 513, 496, 484, 518, 538, 13, 7, 0, 0, 0, 0, 0, 0, 505, 512, 477, 520, 547, 535, 552, 514, 11, 0, 0, 0, 0, 0, 0, 0, 521, 490, 526, 475, 550, 470, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 521, 507, 542, 524, 488, 529, 480, 494, 5, 0, 0, 0, 0, 0, 0, 0, 524, 492, 502, 548, 489, 524, 500, 543, 9, 3, 0, 0, 0, 0, 0, 0, 482, 518, 513, 511, 498, 500, 497, 508, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4085, 4105, 4173, 4072, 4090, 4134, 4048] avg=4096.0000 max=4173 max_rank=3 min=4048 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[481, 453, 511, 493, 518, 502, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 530, 506, 544, 482, 512, 520, 517, 470, 0, 0, 0, 0, 0, 0, 0, 0, 488, 538, 522, 527, 507, 509, 465, 483, 14, 6, 0, 0, 0, 0, 0, 0, 510, 535, 516, 560, 534, 488, 477, 511, 12, 0, 0, 0, 0, 0, 0, 0, 550, 516, 495, 518, 503, 508, 542, 537, 0, 0, 0, 0, 0, 0, 0, 0, 537, 507, 489, 496, 504, 527, 535, 516, 6, 0, 0, 0, 0, 0, 0, 0, 522, 483, 550, 507, 489, 527, 536, 510, 10, 3, 0, 0, 0, 0, 0, 0, 529, 499, 509, 495, 495, 498, 538, 490, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3989, 4081, 4059, 4143, 4169, 4117, 4137, 4073] avg=4096.0000 max=4169 max_rank=4 min=3989 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[496, 523, 506, 531, 500, 468, 493, 517, 0, 0, 0, 0, 0, 0, 0, 0, 531, 501, 528, 492, 482, 554, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 502, 532, 494, 493, 509, 519, 494, 522, 13, 6, 0, 0, 0, 0, 0, 0, 477, 487, 518, 488, 534, 488, 527, 487, 10, 0, 0, 0, 0, 0, 0, 0, 524, 530, 520, 503, 511, 523, 468, 522, 0, 0, 0, 0, 0, 0, 0, 0, 521, 550, 536, 478, 518, 519, 483, 486, 5, 0, 0, 0, 0, 0, 0, 0, 502, 512, 490, 531, 536, 540, 518, 547, 10, 2, 0, 0, 0, 0, 0, 0, 517, 511, 543, 482, 518, 502, 537, 477, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4034, 4142, 4084, 4016, 4101, 4096, 4188, 4107] avg=4096.0000 max=4188 max_rank=6 min=4016 min_rank=3 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[493, 547, 535, 538, 523, 477, 551, 529, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 509, 492, 525, 485, 486, 561, 0, 0, 0, 0, 0, 0, 0, 0, 494, 536, 519, 499, 525, 471, 519, 506, 18, 11, 0, 0, 0, 0, 0, 0, 496, 455, 483, 519, 517, 507, 494, 515, 14, 0, 0, 0, 0, 0, 0, 0, 518, 549, 514, 535, 488, 509, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 519, 533, 518, 535, 505, 510, 497, 494, 8, 0, 0, 0, 0, 0, 0, 0, 504, 496, 512, 542, 497, 478, 511, 520, 14, 7, 0, 0, 0, 0, 0, 0, 535, 468, 492, 493, 503, 510, 525, 516, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4110, 4098, 4000, 4099, 4119, 4081, 4068] avg=4096.0000 max=4193 max_rank=0 min=4000 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[496, 529, 474, 491, 512, 526, 517, 541, 0, 0, 0, 0, 0, 0, 0, 0, 484, 540, 526, 502, 502, 535, 494, 531, 0, 0, 0, 0, 0, 0, 0, 0, 484, 490, 528, 530, 521, 522, 531, 518, 13, 7, 0, 0, 0, 0, 0, 0, 494, 516, 501, 504, 531, 487, 517, 538, 10, 0, 0, 0, 0, 0, 0, 0, 528, 509, 508, 506, 508, 500, 481, 476, 0, 0, 0, 0, 0, 0, 0, 0, 538, 551, 514, 501, 491, 537, 502, 526, 5, 0, 0, 0, 0, 0, 0, 0, 449, 540, 510, 512, 464, 520, 492, 516, 9, 2, 0, 0, 0, 0, 0, 0, 514, 505, 501, 491, 549, 528, 495, 527, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4114, 4144, 4098, 4016, 4165, 4014, 4131] avg=4096.0000 max=4165 max_rank=5 min=4014 min_rank=6 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[524, 528, 509, 525, 505, 534, 524, 503, 481, 491, 566, 502, 530, 498, 560, 521, 546, 493, 514, 532, 497, 507, 537, 504, 483, 527, 527, 507, 503, 509, 508, 486, 536, 477, 512, 493, 482, 538, 504, 530, 523, 523, 509, 510, 550, 518, 482, 483, 497, 501, 518, 487, 486, 477, 544, 499, 495, 555, 465, 527, 497, 512, 516, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[512, 507, 547, 524, 526, 508, 538, 540, 490, 526, 544, 532, 495, 524, 582, 556, 519, 450, 499, 542, 542, 500, 522, 542, 503, 476, 496, 478, 545, 518, 487, 502, 538, 505, 484, 514, 487, 495, 538, 503, 517, 505, 503, 521, 517, 548, 481, 507, 470, 505, 541, 514, 496, 509, 532, 500, 497, 471, 511, 479, 503, 514, 501, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[535, 526, 496, 514, 490, 545, 527, 519, 536, 535, 489, 500, 498, 514, 509, 512, 549, 505, 514, 524, 555, 492, 566, 483, 518, 466, 542, 506, 516, 499, 524, 510, 488, 550, 558, 504, 487, 485, 511, 490, 518, 520, 510, 518, 502, 493, 472, 512, 497, 521, 513, 510, 547, 499, 527, 483, 481, 520, 519, 485, 520, 537, 471, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[503, 538, 537, 492, 493, 524, 500, 513, 510, 494, 548, 539, 549, 509, 501, 492, 535, 493, 510, 470, 506, 509, 498, 493, 539, 487, 528, 513, 500, 489, 510, 556, 504, 482, 495, 522, 517, 540, 520, 523, 561, 516, 531, 511, 521, 504, 516, 499, 535, 476, 491, 531, 508, 505, 500, 495, 531, 480, 522, 517, 510, 512, 491, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[492, 535, 533, 504, 557, 502, 517, 525, 529, 526, 508, 523, 519, 523, 538, 489, 527, 547, 519, 485, 517, 518, 540, 538, 525, 482, 514, 485, 509, 491, 476, 489, 530, 503, 472, 514, 464, 526, 491, 452, 503, 561, 544, 481, 493, 509, 508, 492, 495, 518, 520, 528, 506, 495, 485, 523, 503, 525, 487, 567, 534, 507, 527, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[514, 539, 506, 487, 539, 505, 503, 539, 527, 512, 488, 531, 511, 504, 520, 503, 532, 503, 496, 533, 515, 521, 490, 480, 496, 484, 529, 483, 543, 531, 500, 513, 519, 479, 540, 508, 542, 494, 499, 496, 490, 542, 534, 544, 507, 506, 524, 520, 537, 508, 456, 534, 521, 504, 484, 509, 528, 527, 515, 478, 510, 526, 518, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[503, 486, 523, 518, 512, 545, 503, 501, 541, 530, 505, 509, 506, 528, 536, 493, 503, 523, 537, 483, 503, 542, 522, 537, 471, 522, 524, 506, 513, 499, 489, 520, 497, 494, 471, 557, 548, 526, 548, 509, 486, 509, 513, 510, 507, 484, 486, 490, 480, 506, 523, 475, 526, 535, 510, 501, 454, 545, 519, 512, 538, 515, 532, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[513, 515, 517, 538, 518, 502, 540, 517, 523, 479, 539, 524, 488, 511, 514, 518, 522, 538, 526, 492, 514, 483, 556, 555, 505, 522, 532, 511, 516, 510, 499, 516, 527, 492, 451, 507, 463, 524, 522, 508, 527, 456, 480, 524, 511, 523, 531, 513, 497, 524, 491, 503, 520, 508, 550, 528, 527, 512, 509, 483, 530, 485, 507, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[524, 480, 509, 525, 505, 534, 524, 503, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 566, 502, 530, 498, 514, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 514, 532, 497, 507, 537, 504, 0, 0, 0, 0, 0, 0, 0, 0, 483, 527, 527, 507, 503, 509, 508, 486, 38, 0, 0, 0, 0, 0, 0, 0, 536, 477, 512, 493, 482, 538, 504, 530, 26, 10, 0, 0, 0, 0, 0, 0, 523, 523, 509, 510, 550, 518, 482, 483, 7, 0, 0, 0, 0, 0, 0, 0, 497, 501, 518, 487, 486, 477, 544, 499, 20, 12, 0, 0, 0, 0, 0, 0, 495, 555, 465, 527, 497, 512, 516, 541, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4103, 4096, 4088, 4108, 4105, 4041, 4123] avg=4096.0000 max=4123 max_rank=7 min=4041 min_rank=6 imbalance_ratio=1.006592 max_load_reduction=28645 max_load_reduction_pct=0.874176 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 461, 547, 524, 526, 508, 538, 540, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 544, 532, 495, 524, 534, 556, 0, 0, 0, 0, 0, 0, 0, 0, 487, 450, 499, 542, 542, 500, 522, 542, 0, 0, 0, 0, 0, 0, 0, 0, 503, 476, 496, 478, 545, 518, 487, 502, 37, 0, 0, 0, 0, 0, 0, 0, 538, 505, 484, 514, 487, 495, 538, 503, 27, 9, 0, 0, 0, 0, 0, 0, 517, 505, 503, 521, 517, 548, 481, 507, 7, 0, 0, 0, 0, 0, 0, 0, 470, 505, 541, 514, 496, 509, 532, 500, 21, 11, 0, 0, 0, 0, 0, 0, 497, 471, 511, 479, 503, 514, 501, 490, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4201, 4084, 4042, 4100, 4106, 4099, 3980] avg=4096.0000 max=4201 max_rank=1 min=3980 min_rank=7 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[492, 486, 533, 504, 557, 502, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 529, 526, 508, 523, 519, 523, 494, 489, 0, 0, 0, 0, 0, 0, 0, 0, 494, 547, 519, 485, 517, 518, 540, 538, 0, 0, 0, 0, 0, 0, 0, 0, 525, 482, 514, 485, 509, 491, 476, 489, 39, 0, 0, 0, 0, 0, 0, 0, 530, 503, 472, 514, 464, 526, 491, 452, 25, 10, 0, 0, 0, 0, 0, 0, 503, 561, 544, 481, 493, 509, 508, 492, 7, 0, 0, 0, 0, 0, 0, 0, 495, 518, 520, 528, 506, 495, 485, 523, 19, 11, 0, 0, 0, 0, 0, 0, 503, 525, 487, 567, 534, 507, 527, 523, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4111, 4158, 4010, 3987, 4098, 4100, 4188] avg=4096.0000 max=4188 max_rank=7 min=3987 min_rank=4 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 489, 537, 492, 493, 524, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 548, 539, 549, 509, 460, 492, 0, 0, 0, 0, 0, 0, 0, 0, 501, 493, 510, 470, 506, 509, 498, 493, 0, 0, 0, 0, 0, 0, 0, 0, 539, 487, 528, 513, 500, 489, 510, 556, 39, 0, 0, 0, 0, 0, 0, 0, 504, 482, 495, 522, 517, 540, 520, 523, 23, 10, 0, 0, 0, 0, 0, 0, 561, 516, 531, 511, 521, 504, 516, 499, 7, 0, 0, 0, 0, 0, 0, 0, 535, 476, 491, 531, 508, 505, 500, 495, 18, 12, 0, 0, 0, 0, 0, 0, 531, 480, 522, 517, 510, 512, 491, 524, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4101, 3980, 4161, 4136, 4166, 4071, 4102] avg=4096.0000 max=4166 max_rank=5 min=3980 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[535, 478, 496, 514, 490, 545, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 536, 535, 489, 500, 498, 514, 467, 512, 0, 0, 0, 0, 0, 0, 0, 0, 515, 505, 514, 524, 555, 492, 566, 483, 0, 0, 0, 0, 0, 0, 0, 0, 518, 466, 542, 506, 516, 499, 524, 510, 38, 0, 0, 0, 0, 0, 0, 0, 488, 550, 558, 504, 487, 485, 511, 490, 24, 10, 0, 0, 0, 0, 0, 0, 518, 520, 510, 518, 502, 493, 472, 512, 7, 0, 0, 0, 0, 0, 0, 0, 497, 521, 513, 510, 547, 499, 527, 483, 18, 12, 0, 0, 0, 0, 0, 0, 481, 520, 519, 485, 520, 537, 471, 506, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4051, 4154, 4119, 4107, 4052, 4127, 4054] avg=4096.0000 max=4154 max_rank=2 min=4051 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[503, 442, 523, 518, 512, 545, 503, 501, 0, 0, 0, 0, 0, 0, 0, 0, 541, 530, 505, 509, 506, 528, 492, 493, 0, 0, 0, 0, 0, 0, 0, 0, 471, 523, 537, 483, 503, 542, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 471, 522, 524, 506, 513, 499, 489, 520, 35, 0, 0, 0, 0, 0, 0, 0, 497, 494, 471, 557, 548, 526, 548, 509, 25, 9, 0, 0, 0, 0, 0, 0, 486, 509, 513, 510, 507, 484, 486, 490, 7, 0, 0, 0, 0, 0, 0, 0, 480, 506, 523, 475, 526, 535, 510, 501, 19, 11, 0, 0, 0, 0, 0, 0, 454, 545, 519, 512, 538, 515, 532, 529, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4104, 4118, 4079, 4184, 3992, 4086, 4158] avg=4096.0000 max=4184 max_rank=4 min=3992 min_rank=5 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 483, 506, 487, 539, 505, 503, 539, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 488, 531, 511, 504, 468, 503, 0, 0, 0, 0, 0, 0, 0, 0, 487, 503, 496, 533, 515, 521, 490, 480, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 529, 483, 543, 531, 500, 513, 42, 0, 0, 0, 0, 0, 0, 0, 519, 479, 540, 508, 542, 494, 499, 496, 28, 14, 0, 0, 0, 0, 0, 0, 490, 542, 534, 544, 507, 506, 524, 520, 10, 0, 0, 0, 0, 0, 0, 0, 537, 508, 456, 534, 521, 504, 484, 509, 24, 16, 0, 0, 0, 0, 0, 0, 528, 527, 515, 478, 510, 526, 518, 492, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4044, 4025, 4121, 4119, 4177, 4093, 4113] avg=4096.0000 max=4177 max_rank=5 min=4025 min_rank=2 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 469, 517, 538, 518, 502, 540, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 479, 539, 524, 488, 511, 471, 518, 0, 0, 0, 0, 0, 0, 0, 0, 489, 538, 526, 492, 514, 483, 556, 555, 0, 0, 0, 0, 0, 0, 0, 0, 505, 522, 532, 511, 516, 510, 499, 516, 37, 0, 0, 0, 0, 0, 0, 0, 527, 492, 451, 507, 463, 524, 522, 508, 24, 9, 0, 0, 0, 0, 0, 0, 527, 456, 480, 524, 511, 523, 531, 513, 7, 0, 0, 0, 0, 0, 0, 0, 497, 524, 491, 503, 520, 508, 550, 528, 19, 11, 0, 0, 0, 0, 0, 0, 527, 512, 509, 483, 530, 485, 507, 482, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4053, 4153, 4148, 4027, 4072, 4151, 4050] avg=4096.0000 max=4153 max_rank=2 min=4027 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3840, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4096, 4224, 4096, 4352, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[529, 524, 483, 512, 510, 476, 525, 557, 511, 497, 518, 521, 551, 498, 564, 500, 507, 507, 497, 509, 486, 522, 510, 554, 501, 505, 477, 513, 525, 505, 522, 505, 497, 511, 501, 513, 508, 522, 453, 506, 517, 537, 526, 533, 518, 540, 557, 513, 479, 517, 539, 501, 485, 494, 482, 539, 527, 487, 501, 522, 505, 485, 513, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[519, 525, 521, 558, 475, 495, 546, 528, 490, 508, 552, 487, 485, 512, 477, 524, 520, 495, 559, 498, 487, 499, 504, 465, 530, 505, 493, 505, 523, 500, 511, 496, 513, 534, 545, 520, 542, 531, 508, 520, 541, 484, 493, 557, 525, 502, 492, 509, 530, 528, 497, 487, 495, 535, 538, 500, 512, 514, 507, 537, 491, 489, 506, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[514, 553, 533, 489, 518, 525, 489, 515, 508, 526, 543, 494, 491, 516, 518, 512, 547, 508, 523, 542, 545, 500, 487, 514, 516, 484, 486, 529, 511, 498, 522, 472, 522, 496, 530, 515, 501, 514, 532, 536, 496, 501, 523, 517, 505, 514, 509, 465, 538, 490, 513, 521, 502, 516, 578, 517, 527, 488, 495, 467, 512, 501, 510, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[517, 494, 529, 566, 511, 504, 506, 556, 547, 538, 515, 515, 519, 502, 490, 524, 548, 507, 488, 516, 514, 478, 514, 530, 526, 530, 507, 536, 511, 470, 503, 500, 530, 479, 540, 511, 510, 542, 505, 518, 483, 494, 520, 509, 511, 473, 494, 553, 499, 482, 518, 518, 475, 486, 515, 514, 537, 513, 500, 477, 542, 519, 526, 464] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[491, 512, 488, 535, 503, 490, 516, 512, 516, 523, 486, 513, 506, 531, 518, 528, 538, 545, 509, 522, 523, 490, 517, 515, 517, 506, 491, 513, 496, 473, 518, 536, 512, 473, 494, 494, 488, 509, 529, 519, 511, 515, 518, 518, 463, 525, 526, 507, 529, 525, 515, 479, 540, 540, 508, 527, 527, 488, 500, 524, 530, 534, 528, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[457, 537, 492, 543, 519, 527, 518, 506, 520, 513, 515, 528, 550, 514, 518, 528, 538, 540, 492, 494, 517, 513, 518, 450, 543, 502, 520, 504, 539, 538, 518, 524, 515, 498, 508, 490, 476, 506, 534, 469, 534, 505, 515, 500, 528, 533, 504, 541, 528, 530, 499, 510, 505, 478, 536, 502, 523, 468, 508, 490, 488, 474, 507, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[547, 534, 503, 508, 541, 506, 508, 505, 502, 488, 559, 508, 472, 537, 536, 488, 536, 519, 509, 484, 512, 500, 515, 509, 493, 506, 516, 532, 519, 526, 502, 527, 553, 520, 505, 491, 518, 547, 485, 501, 460, 515, 529, 541, 525, 510, 468, 532, 505, 488, 483, 513, 502, 554, 501, 508, 517, 496, 501, 501, 515, 490, 540, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[508, 507, 505, 508, 559, 546, 511, 562, 517, 504, 492, 490, 494, 514, 493, 482, 479, 533, 548, 518, 508, 488, 485, 513, 519, 539, 522, 508, 483, 521, 481, 521, 508, 529, 521, 506, 532, 519, 513, 463, 495, 524, 501, 530, 538, 526, 541, 473, 497, 497, 502, 466, 502, 499, 524, 527, 546, 552, 471, 540, 530, 514, 496, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[529, 524, 483, 512, 510, 476, 525, 514, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 511, 521, 551, 498, 564, 500, 0, 0, 0, 0, 0, 0, 0, 0, 507, 507, 497, 509, 486, 522, 510, 554, 1, 0, 0, 0, 0, 0, 0, 0, 501, 505, 477, 513, 525, 505, 522, 505, 10, 2, 0, 0, 0, 0, 0, 0, 497, 511, 501, 513, 508, 522, 453, 506, 4, 0, 0, 0, 0, 0, 0, 0, 517, 537, 526, 522, 518, 540, 557, 513, 0, 0, 0, 0, 0, 0, 0, 0, 479, 517, 539, 501, 485, 494, 482, 539, 13, 1, 0, 0, 0, 0, 0, 0, 527, 487, 501, 522, 505, 485, 513, 519, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4153, 4093, 4065, 4015, 4230, 4050, 4089] avg=4096.0000 max=4230 max_rank=5 min=4015 min_rank=4 imbalance_ratio=1.032715 max_load_reduction=28538 max_load_reduction_pct=0.870911 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[514, 553, 533, 489, 518, 525, 489, 475, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 536, 494, 491, 516, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 547, 508, 523, 542, 545, 500, 487, 514, 1, 0, 0, 0, 0, 0, 0, 0, 516, 484, 486, 529, 511, 498, 522, 472, 9, 2, 0, 0, 0, 0, 0, 0, 522, 496, 530, 515, 501, 514, 532, 536, 4, 0, 0, 0, 0, 0, 0, 0, 496, 501, 523, 507, 505, 514, 509, 465, 0, 0, 0, 0, 0, 0, 0, 0, 538, 490, 513, 521, 502, 516, 578, 517, 12, 1, 0, 0, 0, 0, 0, 0, 527, 488, 495, 467, 512, 501, 510, 489, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4101, 4167, 4029, 4150, 4020, 4188, 4017] avg=4096.0000 max=4188 max_rank=6 min=4017 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 494, 529, 566, 511, 504, 506, 513, 0, 0, 0, 0, 0, 0, 0, 0, 547, 538, 508, 515, 519, 502, 490, 524, 0, 0, 0, 0, 0, 0, 0, 0, 548, 507, 488, 516, 514, 478, 514, 530, 1, 0, 0, 0, 0, 0, 0, 0, 526, 530, 507, 536, 511, 470, 503, 500, 9, 2, 0, 0, 0, 0, 0, 0, 530, 479, 540, 511, 510, 542, 505, 518, 4, 0, 0, 0, 0, 0, 0, 0, 483, 494, 520, 499, 511, 473, 494, 553, 0, 0, 0, 0, 0, 0, 0, 0, 499, 482, 518, 518, 475, 486, 515, 514, 13, 1, 0, 0, 0, 0, 0, 0, 537, 513, 500, 477, 542, 519, 526, 464, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4143, 4096, 4094, 4139, 4027, 4021, 4108] avg=4096.0000 max=4143 max_rank=1 min=4021 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 525, 521, 558, 475, 495, 546, 487, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 544, 487, 485, 512, 477, 524, 0, 0, 0, 0, 0, 0, 0, 0, 520, 495, 559, 498, 487, 499, 504, 465, 1, 0, 0, 0, 0, 0, 0, 0, 530, 505, 493, 505, 523, 500, 511, 496, 10, 2, 0, 0, 0, 0, 0, 0, 513, 534, 545, 520, 542, 531, 508, 520, 5, 0, 0, 0, 0, 0, 0, 0, 541, 484, 493, 546, 525, 502, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 530, 528, 497, 487, 495, 535, 538, 500, 13, 1, 0, 0, 0, 0, 0, 0, 512, 514, 507, 537, 491, 489, 506, 494, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4027, 4028, 4075, 4218, 4092, 4124, 4078] avg=4096.0000 max=4218 max_rank=4 min=4027 min_rank=1 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 512, 488, 535, 503, 490, 516, 472, 0, 0, 0, 0, 0, 0, 0, 0, 516, 523, 479, 513, 506, 531, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 538, 545, 509, 522, 523, 490, 517, 515, 1, 0, 0, 0, 0, 0, 0, 0, 517, 506, 491, 513, 496, 473, 518, 536, 9, 2, 0, 0, 0, 0, 0, 0, 512, 473, 494, 494, 488, 509, 529, 519, 4, 0, 0, 0, 0, 0, 0, 0, 511, 515, 518, 508, 463, 525, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 515, 479, 540, 540, 508, 527, 12, 1, 0, 0, 0, 0, 0, 0, 527, 488, 500, 524, 530, 534, 528, 496, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4114, 4160, 4061, 4022, 4073, 4176, 4155] avg=4096.0000 max=4176 max_rank=6 min=4007 min_rank=0 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 507, 505, 508, 559, 546, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 504, 485, 490, 494, 514, 493, 482, 0, 0, 0, 0, 0, 0, 0, 0, 479, 533, 548, 518, 508, 488, 485, 513, 1, 0, 0, 0, 0, 0, 0, 0, 519, 539, 522, 508, 483, 521, 481, 521, 9, 2, 0, 0, 0, 0, 0, 0, 508, 529, 521, 506, 532, 519, 513, 463, 4, 0, 0, 0, 0, 0, 0, 0, 495, 524, 501, 520, 538, 526, 541, 473, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 502, 466, 502, 499, 524, 527, 14, 1, 0, 0, 0, 0, 0, 0, 546, 552, 471, 540, 530, 514, 496, 528, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4162, 3979, 4073, 4105, 4095, 4118, 4029, 4207] avg=4096.0000 max=4207 max_rank=7 min=3979 min_rank=1 imbalance_ratio=1.027100 max_load_reduction=28561 max_load_reduction_pct=0.871613 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[457, 537, 492, 543, 519, 527, 518, 467, 0, 0, 0, 0, 0, 0, 0, 0, 520, 513, 508, 528, 550, 514, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 538, 540, 492, 494, 517, 513, 518, 450, 1, 0, 0, 0, 0, 0, 0, 0, 543, 502, 520, 504, 539, 538, 518, 524, 9, 2, 0, 0, 0, 0, 0, 0, 515, 498, 508, 490, 476, 506, 534, 469, 4, 0, 0, 0, 0, 0, 0, 0, 534, 505, 515, 490, 528, 533, 504, 541, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 499, 510, 505, 478, 536, 502, 12, 1, 0, 0, 0, 0, 0, 0, 523, 468, 508, 490, 488, 474, 507, 531, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4179, 4063, 4199, 4000, 4150, 4101, 4016] avg=4096.0000 max=4199 max_rank=3 min=4000 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16267 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[547, 534, 503, 508, 541, 506, 508, 457, 0, 0, 0, 0, 0, 0, 0, 0, 502, 488, 541, 508, 472, 537, 536, 488, 0, 0, 0, 0, 0, 0, 0, 0, 536, 519, 509, 484, 512, 500, 515, 509, 4, 0, 0, 0, 0, 0, 0, 0, 493, 506, 516, 532, 519, 526, 502, 527, 14, 5, 0, 0, 0, 0, 0, 0, 553, 520, 505, 491, 518, 547, 485, 501, 9, 0, 0, 0, 0, 0, 0, 0, 460, 515, 529, 519, 525, 510, 468, 532, 0, 0, 0, 0, 0, 0, 0, 0, 505, 488, 483, 513, 502, 554, 501, 508, 17, 8, 0, 0, 0, 0, 0, 0, 517, 496, 501, 501, 515, 490, 540, 507, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4072, 4088, 4140, 4129, 4058, 4079, 4098] avg=4096.0000 max=4140 max_rank=3 min=4058 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[481, 536, 506, 495, 496, 549, 513, 528, 495, 491, 511, 479, 572, 507, 505, 523, 532, 495, 509, 558, 519, 479, 524, 490, 509, 503, 522, 519, 508, 492, 511, 541, 518, 506, 473, 508, 492, 508, 546, 498, 506, 524, 549, 526, 520, 523, 461, 531, 481, 509, 542, 517, 495, 512, 519, 525, 529, 537, 489, 483, 516, 521, 509, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[519, 507, 509, 506, 539, 518, 486, 512, 499, 510, 523, 541, 519, 503, 483, 501, 525, 491, 502, 500, 519, 499, 496, 534, 550, 522, 536, 505, 520, 524, 499, 507, 513, 516, 516, 524, 517, 508, 559, 550, 561, 548, 479, 511, 533, 513, 481, 476, 545, 488, 515, 521, 507, 496, 500, 494, 494, 490, 483, 529, 470, 514, 524, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[513, 517, 530, 516, 503, 531, 532, 512, 509, 516, 503, 496, 538, 519, 523, 510, 524, 545, 519, 491, 499, 517, 504, 509, 504, 485, 498, 493, 505, 492, 484, 470, 504, 505, 510, 521, 493, 484, 521, 515, 513, 536, 531, 512, 479, 525, 496, 535, 490, 503, 544, 520, 509, 504, 512, 540, 518, 527, 503, 554, 515, 528, 465, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[505, 512, 530, 512, 525, 517, 506, 498, 499, 473, 505, 529, 527, 509, 497, 548, 522, 498, 501, 464, 522, 484, 497, 505, 560, 546, 532, 504, 500, 525, 521, 513, 511, 502, 498, 476, 519, 529, 510, 518, 485, 457, 584, 539, 503, 527, 540, 488, 487, 510, 458, 501, 499, 512, 490, 536, 546, 510, 515, 522, 551, 500, 550, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[512, 535, 499, 529, 487, 504, 524, 505, 513, 526, 546, 528, 539, 522, 497, 569, 522, 483, 491, 486, 483, 541, 520, 503, 531, 496, 494, 506, 555, 500, 501, 506, 514, 478, 492, 487, 485, 529, 533, 485, 510, 544, 534, 509, 519, 499, 475, 521, 529, 535, 501, 527, 466, 516, 562, 524, 465, 496, 498, 526, 507, 542, 492, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[518, 530, 534, 534, 485, 542, 486, 514, 557, 511, 518, 538, 499, 519, 509, 507, 510, 483, 539, 505, 494, 517, 526, 507, 542, 532, 537, 513, 541, 508, 486, 474, 530, 520, 523, 519, 553, 496, 509, 509, 505, 456, 511, 520, 476, 501, 509, 522, 494, 492, 547, 493, 473, 499, 499, 495, 519, 546, 500, 528, 499, 510, 474, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[507, 521, 538, 512, 535, 530, 538, 536, 569, 516, 508, 516, 494, 529, 509, 508, 504, 480, 486, 533, 503, 477, 522, 516, 474, 506, 536, 503, 523, 531, 478, 546, 497, 480, 525, 522, 495, 492, 513, 501, 539, 520, 542, 527, 478, 504, 482, 526, 480, 520, 525, 520, 491, 498, 535, 511, 508, 497, 511, 500, 508, 491, 535, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[510, 489, 515, 502, 519, 544, 499, 524, 490, 517, 516, 515, 516, 524, 547, 501, 513, 524, 542, 502, 505, 494, 538, 519, 569, 527, 484, 504, 516, 493, 529, 462, 491, 517, 521, 516, 557, 537, 523, 534, 528, 500, 523, 527, 530, 546, 545, 470, 508, 536, 501, 444, 480, 470, 479, 550, 468, 507, 513, 519, 501, 473, 518, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[481, 536, 506, 495, 496, 518, 513, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 491, 511, 479, 536, 507, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0, 532, 495, 509, 558, 519, 479, 524, 490, 29, 0, 0, 0, 0, 0, 0, 0, 505, 503, 522, 519, 508, 492, 511, 541, 0, 0, 0, 0, 0, 0, 0, 0, 518, 506, 473, 508, 492, 508, 546, 498, 2, 0, 0, 0, 0, 0, 0, 0, 506, 524, 534, 526, 520, 523, 461, 531, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 542, 517, 495, 512, 519, 525, 36, 2, 0, 0, 0, 0, 0, 0, 529, 537, 489, 483, 516, 521, 509, 497, 15, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4047, 4135, 4101, 4051, 4125, 4138, 4098] avg=4096.0000 max=4138 max_rank=6 min=4047 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[518, 530, 534, 534, 485, 511, 486, 514, 0, 0, 0, 0, 0, 0, 0, 0, 557, 511, 518, 538, 468, 519, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 539, 505, 494, 517, 526, 507, 29, 0, 0, 0, 0, 0, 0, 0, 538, 532, 537, 513, 541, 508, 486, 474, 0, 0, 0, 0, 0, 0, 0, 0, 530, 520, 523, 519, 553, 496, 509, 509, 2, 0, 0, 0, 0, 0, 0, 0, 505, 456, 497, 520, 476, 501, 509, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 492, 547, 493, 473, 499, 499, 495, 31, 2, 0, 0, 0, 0, 0, 0, 519, 546, 500, 528, 499, 510, 474, 526, 14, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4127, 4110, 4129, 4161, 3986, 4025, 4118] avg=4096.0000 max=4161 max_rank=4 min=3986 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[505, 512, 530, 512, 525, 488, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 473, 505, 529, 494, 509, 497, 548, 0, 0, 0, 0, 0, 0, 0, 0, 522, 498, 501, 464, 522, 484, 497, 505, 27, 0, 0, 0, 0, 0, 0, 0, 556, 546, 532, 504, 500, 525, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 498, 476, 519, 529, 510, 518, 2, 0, 0, 0, 0, 0, 0, 0, 485, 457, 568, 539, 503, 527, 540, 488, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 458, 501, 499, 512, 490, 536, 33, 2, 0, 0, 0, 0, 0, 0, 546, 510, 515, 522, 551, 500, 550, 509, 16, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4054, 4020, 4197, 4065, 4107, 4028, 4221] avg=4096.0000 max=4221 max_rank=7 min=4020 min_rank=2 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 517, 530, 516, 503, 501, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 503, 496, 504, 519, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 524, 545, 519, 491, 499, 517, 504, 509, 28, 0, 0, 0, 0, 0, 0, 0, 500, 485, 498, 493, 505, 492, 484, 470, 0, 0, 0, 0, 0, 0, 0, 0, 504, 505, 510, 521, 493, 484, 521, 515, 2, 0, 0, 0, 0, 0, 0, 0, 513, 536, 516, 512, 479, 525, 496, 535, 0, 0, 0, 0, 0, 0, 0, 0, 490, 503, 544, 520, 509, 504, 512, 540, 34, 2, 0, 0, 0, 0, 0, 0, 518, 527, 503, 554, 515, 528, 465, 549, 15, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4080, 4136, 3927, 4055, 4112, 4158, 4176] avg=4096.0000 max=4176 max_rank=7 min=3927 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 499, 529, 487, 475, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 513, 526, 546, 528, 505, 522, 497, 569, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 491, 486, 483, 541, 520, 503, 27, 0, 0, 0, 0, 0, 0, 0, 527, 496, 494, 506, 555, 500, 501, 506, 0, 0, 0, 0, 0, 0, 0, 0, 514, 478, 492, 487, 485, 529, 533, 485, 2, 0, 0, 0, 0, 0, 0, 0, 510, 544, 519, 509, 519, 499, 475, 521, 0, 0, 0, 0, 0, 0, 0, 0, 529, 535, 501, 527, 466, 516, 562, 524, 34, 2, 0, 0, 0, 0, 0, 0, 465, 496, 498, 526, 507, 542, 492, 515, 15, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4206, 4056, 4085, 4005, 4096, 4196, 4058] avg=4096.0000 max=4206 max_rank=1 min=4005 min_rank=4 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16355 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[507, 521, 538, 512, 535, 500, 538, 536, 0, 0, 0, 0, 0, 0, 0, 0, 569, 516, 508, 516, 463, 529, 509, 508, 0, 0, 0, 0, 0, 0, 0, 0, 504, 480, 486, 533, 503, 477, 522, 516, 28, 0, 0, 0, 0, 0, 0, 0, 471, 506, 536, 503, 523, 531, 478, 546, 0, 0, 0, 0, 0, 0, 0, 0, 497, 480, 525, 522, 495, 492, 513, 501, 1, 0, 0, 0, 0, 0, 0, 0, 539, 520, 527, 527, 478, 504, 482, 526, 0, 0, 0, 0, 0, 0, 0, 0, 480, 520, 525, 520, 491, 498, 535, 511, 31, 2, 0, 0, 0, 0, 0, 0, 508, 497, 511, 500, 508, 491, 535, 511, 15, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4187, 4118, 4049, 4094, 4026, 4103, 4113, 4078] avg=4096.0000 max=4187 max_rank=0 min=4026 min_rank=4 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 507, 509, 506, 539, 489, 486, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 510, 523, 541, 486, 503, 483, 501, 0, 0, 0, 0, 0, 0, 0, 0, 525, 491, 502, 500, 519, 499, 496, 534, 27, 0, 0, 0, 0, 0, 0, 0, 546, 522, 536, 505, 520, 524, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 516, 524, 517, 508, 559, 550, 2, 0, 0, 0, 0, 0, 0, 0, 561, 548, 466, 511, 533, 513, 481, 476, 0, 0, 0, 0, 0, 0, 0, 0, 545, 488, 515, 521, 507, 496, 500, 494, 33, 2, 0, 0, 0, 0, 0, 0, 494, 490, 483, 529, 470, 514, 524, 489, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4046, 4093, 4159, 4205, 4089, 4101, 4008] avg=4096.0000 max=4205 max_rank=4 min=4008 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[510, 489, 515, 502, 519, 505, 499, 524, 0, 0, 0, 0, 0, 0, 0, 0, 490, 517, 516, 515, 480, 524, 547, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 542, 502, 505, 494, 538, 519, 32, 0, 0, 0, 0, 0, 0, 0, 561, 527, 484, 504, 516, 493, 529, 462, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 521, 516, 557, 537, 523, 534, 4, 0, 0, 0, 0, 0, 0, 0, 528, 500, 504, 527, 530, 546, 545, 470, 0, 0, 0, 0, 0, 0, 0, 0, 508, 536, 501, 444, 480, 470, 479, 550, 36, 5, 0, 0, 0, 0, 0, 0, 468, 507, 513, 519, 501, 473, 518, 487, 19, 4, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4090, 4169, 4076, 4200, 4150, 4009, 4011] avg=4096.0000 max=4200 max_rank=4 min=4009 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 3968, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[543, 514, 513, 527, 531, 534, 495, 523, 530, 513, 471, 515, 512, 483, 533, 481, 479, 539, 489, 521, 478, 501, 507, 526, 486, 500, 551, 517, 539, 515, 522, 536, 489, 497, 497, 548, 492, 515, 526, 488, 504, 536, 486, 503, 562, 515, 492, 507, 480, 533, 547, 483, 508, 532, 472, 499, 532, 525, 504, 558, 519, 479, 497, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[519, 524, 493, 517, 501, 515, 518, 515, 517, 497, 508, 536, 513, 509, 498, 510, 508, 479, 515, 471, 469, 488, 500, 527, 528, 538, 533, 547, 518, 508, 519, 518, 524, 524, 498, 488, 532, 480, 522, 498, 546, 530, 552, 541, 473, 489, 506, 512, 531, 509, 488, 502, 511, 521, 486, 486, 529, 505, 513, 491, 527, 523, 537, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[543, 506, 504, 496, 497, 537, 512, 492, 532, 506, 541, 500, 548, 554, 542, 533, 506, 507, 510, 524, 500, 520, 498, 478, 495, 507, 511, 531, 476, 511, 466, 518, 478, 519, 485, 520, 528, 503, 520, 541, 505, 520, 539, 504, 487, 513, 498, 501, 511, 476, 510, 502, 492, 549, 514, 523, 480, 509, 494, 526, 522, 537, 530, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[524, 518, 524, 523, 490, 530, 482, 527, 527, 531, 541, 474, 501, 486, 545, 522, 527, 503, 511, 502, 518, 542, 490, 495, 486, 508, 469, 496, 525, 482, 541, 508, 479, 498, 499, 503, 522, 508, 528, 534, 550, 501, 496, 540, 519, 521, 515, 478, 494, 504, 493, 497, 511, 520, 498, 531, 532, 560, 481, 508, 540, 531, 512, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[515, 525, 516, 519, 516, 514, 558, 524, 498, 502, 527, 510, 552, 504, 484, 540, 522, 462, 522, 532, 517, 504, 546, 545, 495, 527, 540, 510, 544, 511, 501, 549, 501, 511, 523, 515, 530, 483, 527, 522, 513, 497, 472, 503, 522, 504, 486, 498, 524, 509, 488, 488, 518, 520, 510, 477, 484, 491, 481, 490, 513, 510, 519, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[509, 501, 523, 527, 494, 516, 559, 511, 499, 496, 499, 504, 524, 555, 529, 511, 519, 517, 449, 471, 551, 478, 500, 504, 529, 493, 530, 481, 537, 482, 546, 454, 506, 497, 541, 512, 485, 521, 523, 516, 501, 521, 494, 537, 504, 502, 505, 497, 519, 498, 574, 509, 555, 532, 494, 525, 526, 504, 513, 513, 508, 522, 486, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[521, 516, 527, 503, 533, 536, 524, 550, 542, 548, 486, 497, 504, 509, 545, 569, 505, 508, 490, 503, 481, 493, 547, 484, 514, 494, 503, 510, 472, 511, 530, 548, 548, 500, 522, 501, 522, 515, 518, 526, 489, 485, 492, 526, 486, 503, 512, 531, 490, 518, 522, 495, 541, 547, 493, 518, 492, 530, 513, 529, 474, 496, 475, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[503, 509, 531, 529, 510, 522, 538, 475, 504, 483, 486, 516, 502, 540, 512, 535, 499, 557, 528, 500, 485, 537, 509, 487, 537, 534, 474, 509, 523, 527, 493, 516, 528, 511, 479, 510, 515, 519, 521, 513, 535, 530, 505, 518, 537, 501, 543, 517, 507, 528, 495, 523, 496, 475, 553, 513, 487, 492, 482, 501, 497, 546, 464, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 514, 513, 527, 531, 487, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0, 530, 513, 471, 515, 512, 483, 533, 455, 0, 0, 0, 0, 0, 0, 0, 0, 479, 539, 489, 521, 478, 501, 507, 526, 47, 1, 0, 0, 0, 0, 0, 0, 486, 500, 551, 517, 539, 515, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 489, 497, 497, 548, 492, 515, 526, 488, 3, 0, 0, 0, 0, 0, 0, 0, 504, 536, 486, 503, 562, 515, 492, 507, 4, 2, 0, 0, 0, 0, 0, 0, 480, 533, 547, 483, 508, 532, 472, 499, 11, 0, 0, 0, 0, 0, 0, 0, 532, 525, 504, 558, 519, 479, 497, 519, 12, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4012, 4088, 4159, 4055, 4111, 4065, 4145] avg=4096.0000 max=4159 max_rank=3 min=4012 min_rank=1 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 506, 504, 496, 497, 490, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 541, 500, 548, 554, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 510, 524, 500, 520, 498, 478, 47, 1, 0, 0, 0, 0, 0, 0, 495, 507, 511, 531, 476, 511, 466, 512, 0, 0, 0, 0, 0, 0, 0, 0, 478, 519, 485, 520, 528, 503, 520, 541, 2, 0, 0, 0, 0, 0, 0, 0, 505, 520, 539, 504, 487, 513, 498, 501, 4, 3, 0, 0, 0, 0, 0, 0, 511, 476, 510, 502, 492, 549, 514, 523, 12, 0, 0, 0, 0, 0, 0, 0, 480, 509, 494, 526, 522, 537, 530, 531, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4226, 4091, 4009, 4096, 4074, 4089, 4143] avg=4096.0000 max=4226 max_rank=1 min=4009 min_rank=3 imbalance_ratio=1.031738 max_load_reduction=28542 max_load_reduction_pct=0.871033 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 525, 516, 519, 516, 469, 558, 524, 0, 0, 0, 0, 0, 0, 0, 0, 498, 502, 527, 510, 552, 504, 484, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 462, 522, 532, 517, 504, 546, 545, 45, 1, 0, 0, 0, 0, 0, 0, 495, 527, 540, 510, 544, 511, 501, 542, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 523, 515, 530, 483, 527, 522, 3, 0, 0, 0, 0, 0, 0, 0, 513, 497, 472, 503, 522, 504, 486, 498, 4, 3, 0, 0, 0, 0, 0, 0, 524, 509, 488, 488, 518, 520, 510, 477, 12, 0, 0, 0, 0, 0, 0, 0, 484, 491, 481, 490, 513, 510, 519, 508, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4087, 4196, 4170, 4115, 4002, 4046, 4010] avg=4096.0000 max=4196 max_rank=2 min=4002 min_rank=5 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16173 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[519, 524, 493, 517, 501, 470, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 497, 508, 536, 513, 509, 498, 482, 0, 0, 0, 0, 0, 0, 0, 0, 508, 479, 515, 471, 469, 488, 500, 527, 45, 1, 0, 0, 0, 0, 0, 0, 528, 538, 533, 547, 518, 508, 519, 512, 0, 0, 0, 0, 0, 0, 0, 0, 524, 524, 498, 488, 532, 480, 522, 498, 2, 0, 0, 0, 0, 0, 0, 0, 546, 530, 552, 541, 473, 489, 506, 512, 4, 2, 0, 0, 0, 0, 0, 0, 531, 509, 488, 502, 511, 521, 486, 486, 12, 0, 0, 0, 0, 0, 0, 0, 529, 505, 513, 491, 527, 523, 537, 538, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4060, 4003, 4203, 4068, 4155, 4046, 4176] avg=4096.0000 max=4203 max_rank=3 min=4003 min_rank=2 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 501, 523, 527, 494, 471, 559, 511, 0, 0, 0, 0, 0, 0, 0, 0, 499, 496, 499, 504, 524, 555, 529, 483, 0, 0, 0, 0, 0, 0, 0, 0, 519, 517, 449, 471, 551, 478, 500, 504, 45, 1, 0, 0, 0, 0, 0, 0, 529, 493, 530, 481, 537, 482, 546, 448, 0, 0, 0, 0, 0, 0, 0, 0, 506, 497, 541, 512, 485, 521, 523, 516, 2, 0, 0, 0, 0, 0, 0, 0, 501, 521, 494, 537, 504, 502, 505, 497, 4, 2, 0, 0, 0, 0, 0, 0, 519, 498, 574, 509, 555, 532, 494, 525, 12, 0, 0, 0, 0, 0, 0, 0, 526, 504, 513, 513, 508, 522, 486, 530, 13, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4089, 4035, 4046, 4103, 4067, 4218, 4115] avg=4096.0000 max=4218 max_rank=6 min=4035 min_rank=2 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[503, 509, 531, 529, 510, 476, 538, 475, 0, 0, 0, 0, 0, 0, 0, 0, 504, 483, 486, 516, 502, 540, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 499, 557, 528, 500, 485, 537, 509, 487, 46, 1, 0, 0, 0, 0, 0, 0, 537, 534, 474, 509, 523, 527, 493, 510, 0, 0, 0, 0, 0, 0, 0, 0, 528, 511, 479, 510, 515, 519, 521, 513, 2, 0, 0, 0, 0, 0, 0, 0, 535, 530, 505, 518, 537, 501, 543, 517, 4, 3, 0, 0, 0, 0, 0, 0, 507, 528, 495, 523, 496, 475, 553, 513, 12, 0, 0, 0, 0, 0, 0, 0, 487, 492, 482, 501, 497, 546, 464, 517, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4048, 4149, 4107, 4098, 4193, 4102, 4000] avg=4096.0000 max=4193 max_rank=5 min=4000 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 518, 524, 523, 490, 483, 482, 527, 0, 0, 0, 0, 0, 0, 0, 0, 527, 531, 541, 474, 501, 486, 545, 493, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 511, 502, 518, 542, 490, 495, 47, 1, 0, 0, 0, 0, 0, 0, 486, 508, 469, 496, 525, 482, 541, 502, 0, 0, 0, 0, 0, 0, 0, 0, 479, 498, 499, 503, 522, 508, 528, 534, 2, 0, 0, 0, 0, 0, 0, 0, 550, 501, 496, 540, 519, 521, 515, 478, 4, 2, 0, 0, 0, 0, 0, 0, 494, 504, 493, 497, 511, 520, 498, 531, 12, 0, 0, 0, 0, 0, 0, 0, 532, 560, 481, 508, 540, 531, 512, 517, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4098, 4136, 4009, 4073, 4126, 4060, 4195] avg=4096.0000 max=4195 max_rank=7 min=4009 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16454 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 516, 527, 503, 533, 485, 524, 550, 0, 0, 0, 0, 0, 0, 0, 0, 542, 548, 486, 497, 504, 509, 545, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 508, 490, 503, 481, 493, 547, 484, 51, 8, 0, 0, 0, 0, 0, 0, 514, 494, 503, 510, 472, 511, 530, 531, 0, 0, 0, 0, 0, 0, 0, 0, 548, 500, 522, 501, 522, 515, 518, 526, 8, 0, 0, 0, 0, 0, 0, 0, 489, 485, 492, 526, 486, 503, 512, 531, 9, 7, 0, 0, 0, 0, 0, 0, 490, 518, 522, 495, 541, 547, 493, 518, 18, 0, 0, 0, 0, 0, 0, 0, 492, 530, 513, 529, 474, 496, 475, 456, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4148, 4070, 4065, 4160, 4040, 4142, 3984] avg=4096.0000 max=4160 max_rank=4 min=3984 min_rank=7 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 14:57:42] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 278.0 | throughput per GPU (TFLOP/s/GPU): 444.2 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.865271E+00 | loss scale: 1.0 | grad norm: 19.074 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[503, 489, 532, 518, 518, 486, 512, 509, 526, 557, 506, 516, 494, 532, 517, 493, 507, 528, 481, 515, 514, 536, 491, 531, 533, 517, 523, 506, 498, 524, 554, 544, 526, 517, 498, 487, 523, 514, 509, 519, 513, 483, 489, 508, 514, 468, 522, 521, 485, 542, 501, 523, 498, 459, 517, 488, 485, 557, 549, 501, 536, 499, 522, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[528, 496, 496, 530, 515, 506, 545, 544, 527, 548, 527, 505, 554, 519, 511, 482, 531, 497, 507, 484, 515, 457, 514, 515, 491, 518, 483, 529, 516, 536, 525, 502, 511, 491, 481, 522, 513, 515, 497, 500, 514, 521, 513, 517, 492, 507, 509, 517, 507, 512, 507, 512, 491, 567, 510, 530, 505, 493, 494, 508, 505, 510, 510, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[522, 509, 485, 532, 503, 457, 525, 540, 523, 511, 532, 465, 500, 562, 510, 541, 477, 514, 485, 486, 531, 512, 529, 514, 538, 552, 520, 514, 509, 516, 484, 513, 508, 545, 472, 510, 523, 510, 527, 488, 539, 573, 496, 500, 497, 523, 529, 494, 492, 486, 549, 489, 493, 536, 478, 525, 517, 531, 516, 492, 517, 468, 489, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[501, 527, 510, 512, 530, 497, 510, 523, 559, 505, 506, 512, 535, 491, 523, 524, 497, 497, 495, 517, 511, 479, 487, 495, 549, 530, 519, 503, 526, 459, 556, 534, 501, 513, 544, 465, 533, 509, 505, 497, 497, 513, 511, 520, 514, 526, 525, 469, 504, 522, 549, 478, 534, 500, 509, 513, 494, 533, 528, 521, 504, 525, 470, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[516, 525, 549, 503, 504, 546, 528, 479, 466, 510, 495, 523, 491, 503, 522, 536, 524, 525, 508, 531, 499, 518, 486, 500, 528, 521, 515, 496, 501, 531, 544, 504, 560, 500, 519, 489, 537, 501, 463, 484, 510, 486, 503, 512, 524, 520, 482, 531, 547, 547, 495, 509, 505, 504, 501, 520, 521, 524, 482, 503, 500, 525, 503, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[494, 551, 524, 493, 522, 511, 544, 518, 498, 485, 524, 484, 512, 493, 486, 495, 517, 519, 540, 493, 487, 543, 508, 533, 548, 542, 492, 548, 478, 514, 508, 528, 530, 534, 473, 517, 508, 536, 541, 513, 510, 512, 522, 520, 495, 533, 514, 499, 506, 502, 539, 470, 481, 523, 515, 520, 497, 517, 515, 501, 501, 486, 488, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[549, 495, 517, 497, 522, 519, 518, 532, 534, 537, 529, 532, 496, 499, 504, 506, 506, 496, 520, 501, 522, 473, 510, 521, 495, 541, 515, 512, 478, 510, 540, 531, 489, 507, 549, 540, 522, 516, 534, 542, 532, 477, 493, 489, 523, 520, 516, 518, 464, 481, 476, 539, 525, 498, 503, 522, 490, 502, 490, 515, 524, 502, 509, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[506, 479, 468, 515, 508, 453, 564, 540, 510, 509, 525, 521, 546, 513, 507, 522, 545, 518, 531, 520, 491, 500, 524, 509, 495, 546, 513, 529, 518, 517, 506, 547, 504, 534, 526, 497, 474, 489, 533, 494, 513, 505, 511, 508, 512, 544, 484, 522, 522, 479, 500, 489, 524, 522, 505, 501, 517, 518, 488, 518, 490, 551, 463, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 551, 524, 493, 522, 511, 528, 518, 0, 0, 0, 0, 0, 0, 0, 0, 498, 467, 524, 484, 512, 493, 486, 495, 0, 0, 0, 0, 0, 0, 0, 0, 517, 519, 540, 493, 487, 543, 508, 533, 25, 0, 0, 0, 0, 0, 0, 0, 548, 488, 492, 548, 478, 514, 508, 528, 0, 0, 0, 0, 0, 0, 0, 0, 530, 534, 473, 517, 508, 536, 541, 513, 5, 0, 0, 0, 0, 0, 0, 0, 510, 512, 522, 520, 495, 533, 514, 499, 11, 0, 0, 0, 0, 0, 0, 0, 506, 502, 539, 470, 481, 523, 515, 520, 28, 0, 0, 0, 0, 0, 0, 0, 497, 517, 515, 501, 501, 486, 488, 518, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4141, 3959, 4165, 4104, 4157, 4116, 4084, 4042] avg=4096.0000 max=4165 max_rank=2 min=3959 min_rank=1 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 489, 532, 518, 518, 486, 498, 509, 0, 0, 0, 0, 0, 0, 0, 0, 526, 537, 506, 516, 494, 532, 517, 493, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 481, 515, 514, 536, 491, 531, 24, 0, 0, 0, 0, 0, 0, 0, 533, 465, 523, 506, 498, 524, 554, 544, 0, 0, 0, 0, 0, 0, 0, 0, 526, 517, 498, 487, 523, 514, 509, 519, 4, 0, 0, 0, 0, 0, 0, 0, 513, 483, 489, 508, 514, 468, 522, 521, 10, 0, 0, 0, 0, 0, 0, 0, 485, 542, 501, 523, 498, 459, 517, 488, 27, 0, 0, 0, 0, 0, 0, 0, 485, 557, 549, 501, 536, 499, 522, 485, 20, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4121, 4127, 4147, 4097, 4028, 4040, 4155] avg=4096.0000 max=4155 max_rank=7 min=4028 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 509, 485, 532, 503, 457, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 523, 492, 532, 465, 500, 562, 510, 541, 0, 0, 0, 0, 0, 0, 0, 0, 477, 514, 485, 486, 531, 512, 529, 514, 26, 0, 0, 0, 0, 0, 0, 0, 538, 496, 520, 514, 509, 516, 484, 513, 0, 0, 0, 0, 0, 0, 0, 0, 508, 545, 472, 510, 523, 510, 527, 488, 4, 0, 0, 0, 0, 0, 0, 0, 539, 573, 496, 500, 497, 523, 529, 494, 11, 0, 0, 0, 0, 0, 0, 0, 492, 486, 549, 489, 493, 536, 478, 525, 29, 0, 0, 0, 0, 0, 0, 0, 517, 531, 516, 492, 517, 468, 489, 545, 19, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4125, 4074, 4090, 4087, 4162, 4077, 4095] avg=4096.0000 max=4162 max_rank=5 min=4058 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[501, 527, 510, 512, 530, 497, 496, 523, 0, 0, 0, 0, 0, 0, 0, 0, 559, 487, 506, 512, 535, 491, 523, 524, 0, 0, 0, 0, 0, 0, 0, 0, 497, 497, 495, 517, 511, 479, 487, 495, 24, 0, 0, 0, 0, 0, 0, 0, 549, 477, 519, 503, 526, 459, 556, 534, 0, 0, 0, 0, 0, 0, 0, 0, 501, 513, 544, 465, 533, 509, 505, 497, 4, 0, 0, 0, 0, 0, 0, 0, 497, 513, 511, 520, 514, 526, 525, 469, 10, 0, 0, 0, 0, 0, 0, 0, 504, 522, 549, 478, 534, 500, 509, 513, 28, 0, 0, 0, 0, 0, 0, 0, 494, 533, 528, 521, 504, 525, 470, 523, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4137, 4002, 4123, 4071, 4085, 4137, 4117] avg=4096.0000 max=4137 max_rank=1 min=4002 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[528, 496, 496, 530, 515, 506, 529, 544, 0, 0, 0, 0, 0, 0, 0, 0, 527, 528, 527, 505, 554, 519, 511, 482, 0, 0, 0, 0, 0, 0, 0, 0, 531, 497, 507, 484, 515, 457, 514, 515, 24, 0, 0, 0, 0, 0, 0, 0, 491, 466, 483, 529, 516, 536, 525, 502, 0, 0, 0, 0, 0, 0, 0, 0, 511, 491, 481, 522, 513, 515, 497, 500, 5, 0, 0, 0, 0, 0, 0, 0, 514, 521, 513, 517, 492, 507, 509, 517, 11, 0, 0, 0, 0, 0, 0, 0, 507, 512, 507, 512, 491, 567, 510, 530, 27, 0, 0, 0, 0, 0, 0, 0, 505, 493, 494, 508, 505, 510, 510, 534, 20, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4153, 4044, 4048, 4035, 4101, 4163, 4080] avg=4096.0000 max=4163 max_rank=6 min=4035 min_rank=4 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 525, 549, 503, 504, 546, 513, 479, 0, 0, 0, 0, 0, 0, 0, 0, 466, 492, 495, 523, 491, 503, 522, 536, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 508, 531, 499, 518, 486, 500, 24, 0, 0, 0, 0, 0, 0, 0, 528, 469, 515, 496, 501, 531, 544, 504, 0, 0, 0, 0, 0, 0, 0, 0, 560, 500, 519, 489, 537, 501, 463, 484, 4, 0, 0, 0, 0, 0, 0, 0, 510, 486, 503, 512, 524, 520, 482, 531, 11, 0, 0, 0, 0, 0, 0, 0, 547, 547, 495, 509, 505, 504, 501, 520, 27, 0, 0, 0, 0, 0, 0, 0, 521, 524, 482, 503, 500, 525, 503, 534, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4028, 4115, 4088, 4057, 4079, 4155, 4111] avg=4096.0000 max=4155 max_rank=6 min=4028 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[549, 495, 517, 497, 522, 519, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 534, 518, 529, 532, 496, 499, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 506, 496, 520, 501, 522, 473, 510, 521, 25, 0, 0, 0, 0, 0, 0, 0, 495, 487, 515, 512, 478, 510, 540, 531, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 549, 540, 522, 516, 534, 542, 4, 0, 0, 0, 0, 0, 0, 0, 532, 477, 493, 489, 523, 520, 516, 518, 11, 0, 0, 0, 0, 0, 0, 0, 464, 481, 476, 539, 525, 498, 503, 522, 28, 0, 0, 0, 0, 0, 0, 0, 490, 502, 490, 515, 524, 502, 509, 504, 19, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4118, 4074, 4068, 4203, 4079, 4036, 4056] avg=4096.0000 max=4203 max_rank=4 min=4036 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 479, 468, 515, 508, 453, 538, 540, 0, 0, 0, 0, 0, 0, 0, 0, 510, 483, 525, 521, 546, 513, 507, 522, 0, 0, 0, 0, 0, 0, 0, 0, 545, 518, 531, 520, 491, 500, 524, 509, 29, 0, 0, 0, 0, 0, 0, 0, 495, 475, 513, 529, 518, 517, 506, 547, 0, 0, 0, 0, 0, 0, 0, 0, 504, 534, 526, 497, 474, 489, 533, 494, 10, 0, 0, 0, 0, 0, 0, 0, 513, 505, 511, 508, 512, 544, 484, 522, 16, 3, 0, 0, 0, 0, 0, 0, 522, 479, 500, 489, 524, 522, 505, 501, 34, 0, 0, 0, 0, 0, 0, 0, 517, 518, 488, 518, 490, 551, 463, 536, 23, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4127, 4167, 4100, 4061, 4118, 4076, 4112] avg=4096.0000 max=4167 max_rank=2 min=4007 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3840, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[553, 475, 513, 542, 533, 505, 515, 509, 501, 505, 502, 527, 534, 538, 522, 517, 547, 511, 507, 514, 498, 501, 526, 538, 509, 526, 468, 527, 496, 510, 504, 481, 508, 540, 534, 491, 486, 523, 496, 496, 522, 512, 522, 494, 510, 529, 527, 500, 494, 486, 536, 536, 503, 499, 513, 475, 509, 495, 538, 497, 489, 541, 517, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[528, 495, 516, 502, 535, 483, 499, 504, 533, 520, 503, 523, 512, 501, 492, 472, 515, 519, 526, 509, 514, 518, 512, 499, 500, 498, 503, 527, 531, 493, 530, 513, 491, 560, 494, 525, 543, 523, 529, 512, 510, 533, 533, 503, 513, 507, 519, 489, 517, 532, 502, 522, 513, 467, 483, 524, 511, 538, 514, 502, 502, 515, 524, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[526, 513, 522, 517, 533, 497, 566, 538, 510, 504, 498, 496, 501, 522, 502, 467, 516, 475, 531, 507, 543, 467, 527, 512, 520, 521, 531, 543, 534, 571, 506, 512, 542, 507, 506, 500, 520, 500, 478, 527, 504, 478, 493, 523, 514, 491, 487, 566, 484, 508, 480, 515, 475, 520, 529, 479, 518, 544, 492, 508, 480, 519, 498, 555] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[508, 566, 521, 556, 539, 495, 517, 527, 516, 536, 496, 486, 522, 464, 529, 517, 511, 505, 506, 505, 514, 525, 548, 536, 526, 499, 518, 494, 490, 538, 500, 528, 550, 533, 510, 487, 554, 519, 500, 513, 502, 534, 530, 523, 514, 494, 489, 516, 506, 526, 490, 497, 515, 479, 482, 486, 488, 486, 496, 524, 471, 476, 510, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[522, 518, 498, 511, 531, 553, 528, 511, 487, 546, 491, 521, 511, 526, 504, 504, 513, 531, 509, 511, 482, 512, 522, 501, 546, 505, 514, 536, 509, 526, 519, 529, 522, 527, 521, 508, 479, 491, 510, 509, 487, 517, 494, 515, 468, 512, 534, 517, 520, 547, 517, 492, 511, 513, 468, 548, 505, 497, 525, 520, 478, 497, 494, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[530, 515, 549, 533, 485, 477, 498, 542, 487, 520, 553, 528, 527, 540, 497, 506, 534, 506, 495, 510, 517, 536, 523, 510, 548, 507, 497, 505, 500, 494, 529, 499, 494, 509, 513, 476, 473, 519, 531, 523, 534, 502, 541, 509, 477, 509, 556, 539, 516, 493, 509, 508, 473, 542, 502, 493, 546, 494, 497, 503, 496, 500, 507, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[469, 502, 502, 519, 508, 518, 525, 499, 524, 490, 502, 525, 504, 507, 528, 556, 524, 534, 498, 487, 545, 516, 500, 498, 529, 547, 517, 519, 502, 531, 501, 514, 519, 538, 532, 516, 484, 527, 491, 480, 494, 525, 529, 543, 506, 549, 511, 550, 516, 508, 492, 515, 506, 498, 498, 489, 509, 500, 502, 520, 489, 491, 507, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[516, 512, 523, 500, 515, 500, 505, 491, 494, 479, 529, 484, 546, 516, 511, 488, 545, 492, 535, 541, 502, 545, 533, 505, 519, 537, 518, 514, 531, 523, 485, 532, 497, 475, 528, 500, 505, 505, 526, 487, 490, 533, 509, 509, 559, 523, 534, 497, 479, 518, 520, 522, 491, 528, 515, 509, 504, 528, 503, 534, 486, 527, 507, 454] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[528, 495, 516, 468, 535, 483, 499, 504, 0, 0, 0, 0, 0, 0, 0, 0, 533, 520, 503, 523, 512, 501, 492, 472, 8, 0, 0, 0, 0, 0, 0, 0, 488, 519, 526, 509, 514, 518, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 470, 498, 503, 527, 531, 493, 530, 513, 0, 0, 0, 0, 0, 0, 0, 0, 491, 560, 494, 525, 543, 523, 529, 512, 6, 0, 0, 0, 0, 0, 0, 0, 510, 533, 533, 503, 513, 507, 519, 468, 0, 0, 0, 0, 0, 0, 0, 0, 517, 532, 502, 522, 513, 467, 483, 524, 34, 18, 0, 0, 0, 0, 0, 0, 511, 538, 514, 502, 502, 515, 524, 493, 27, 12, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4064, 4085, 4065, 4183, 4086, 4112, 4145] avg=4096.0000 max=4183 max_rank=4 min=4028 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16526 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[553, 475, 513, 506, 533, 505, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 502, 527, 534, 538, 522, 517, 8, 0, 0, 0, 0, 0, 0, 0, 518, 511, 507, 514, 498, 501, 526, 538, 0, 0, 0, 0, 0, 0, 0, 0, 479, 526, 468, 527, 496, 510, 504, 481, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 534, 491, 486, 523, 496, 496, 6, 0, 0, 0, 0, 0, 0, 0, 522, 512, 522, 494, 510, 529, 527, 479, 0, 0, 0, 0, 0, 0, 0, 0, 494, 486, 536, 536, 503, 499, 513, 475, 36, 18, 0, 0, 0, 0, 0, 0, 509, 495, 538, 497, 489, 541, 517, 496, 29, 12, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4154, 4113, 3991, 4080, 4095, 4096, 4130] avg=4096.0000 max=4154 max_rank=1 min=3991 min_rank=3 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 566, 521, 519, 539, 495, 517, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 536, 496, 486, 522, 464, 529, 517, 8, 0, 0, 0, 0, 0, 0, 0, 484, 505, 506, 505, 514, 525, 548, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 499, 518, 494, 490, 538, 500, 528, 0, 0, 0, 0, 0, 0, 0, 0, 550, 533, 510, 487, 554, 519, 500, 513, 6, 0, 0, 0, 0, 0, 0, 0, 502, 534, 530, 523, 514, 494, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 506, 526, 490, 497, 515, 479, 482, 486, 37, 19, 0, 0, 0, 0, 0, 0, 488, 486, 496, 524, 471, 476, 510, 530, 27, 13, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4192, 4074, 4123, 4061, 4172, 4081, 4037, 4028] avg=4096.0000 max=4192 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[522, 518, 498, 477, 531, 553, 528, 511, 0, 0, 0, 0, 0, 0, 0, 0, 487, 546, 491, 521, 511, 526, 504, 504, 8, 0, 0, 0, 0, 0, 0, 0, 486, 531, 509, 511, 482, 512, 522, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 514, 536, 509, 526, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 522, 527, 521, 508, 479, 491, 510, 509, 6, 0, 0, 0, 0, 0, 0, 0, 487, 517, 494, 515, 468, 512, 534, 496, 0, 0, 0, 0, 0, 0, 0, 0, 520, 547, 517, 492, 511, 513, 468, 548, 34, 20, 0, 0, 0, 0, 0, 0, 505, 497, 525, 520, 478, 497, 494, 498, 27, 13, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4098, 4054, 4151, 4073, 4023, 4170, 4061] avg=4096.0000 max=4170 max_rank=6 min=4023 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 513, 522, 482, 533, 497, 566, 538, 0, 0, 0, 0, 0, 0, 0, 0, 510, 504, 498, 496, 501, 522, 502, 467, 9, 0, 0, 0, 0, 0, 0, 0, 489, 475, 531, 507, 543, 467, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 488, 521, 531, 543, 534, 571, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 542, 507, 506, 500, 520, 500, 478, 527, 7, 0, 0, 0, 0, 0, 0, 0, 504, 478, 493, 523, 514, 491, 487, 542, 0, 0, 0, 0, 0, 0, 0, 0, 484, 508, 480, 515, 475, 520, 529, 479, 35, 19, 0, 0, 0, 0, 0, 0, 518, 544, 492, 508, 480, 519, 498, 555, 27, 13, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4177, 4009, 4051, 4206, 4087, 4032, 4044, 4162] avg=4096.0000 max=4206 max_rank=3 min=4009 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[469, 502, 502, 484, 508, 518, 525, 499, 0, 0, 0, 0, 0, 0, 0, 0, 524, 490, 502, 525, 504, 507, 528, 556, 9, 0, 0, 0, 0, 0, 0, 0, 496, 534, 498, 487, 545, 516, 500, 498, 0, 0, 0, 0, 0, 0, 0, 0, 497, 547, 517, 519, 502, 531, 501, 514, 0, 0, 0, 0, 0, 0, 0, 0, 519, 538, 532, 516, 484, 527, 491, 480, 7, 0, 0, 0, 0, 0, 0, 0, 494, 525, 529, 543, 506, 549, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 492, 515, 506, 498, 498, 489, 35, 19, 0, 0, 0, 0, 0, 0, 509, 500, 502, 520, 489, 491, 507, 494, 28, 13, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4145, 4074, 4128, 4094, 4184, 4076, 4060] avg=4096.0000 max=4184 max_rank=5 min=4007 min_rank=0 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 515, 549, 493, 485, 477, 498, 542, 0, 0, 0, 0, 0, 0, 0, 0, 487, 520, 553, 528, 527, 540, 497, 506, 11, 0, 0, 0, 0, 0, 0, 0, 502, 506, 495, 510, 517, 536, 523, 510, 0, 0, 0, 0, 0, 0, 0, 0, 508, 507, 497, 505, 500, 494, 529, 499, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 513, 476, 473, 519, 531, 523, 12, 0, 0, 0, 0, 0, 0, 0, 534, 502, 541, 509, 477, 509, 556, 506, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 509, 508, 473, 542, 502, 493, 40, 22, 0, 0, 0, 0, 0, 0, 546, 494, 497, 503, 496, 500, 507, 487, 32, 18, 10, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4169, 4099, 4039, 4050, 4134, 4098, 4090] avg=4096.0000 max=4169 max_rank=1 min=4039 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16372 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 512, 523, 466, 515, 500, 505, 491, 0, 0, 0, 0, 0, 0, 0, 0, 494, 479, 529, 484, 546, 516, 511, 488, 8, 0, 0, 0, 0, 0, 0, 0, 516, 492, 535, 541, 502, 545, 533, 505, 0, 0, 0, 0, 0, 0, 0, 0, 487, 537, 518, 514, 531, 523, 485, 532, 0, 0, 0, 0, 0, 0, 0, 0, 497, 475, 528, 500, 505, 505, 526, 487, 6, 0, 0, 0, 0, 0, 0, 0, 490, 533, 509, 509, 559, 523, 534, 476, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 520, 522, 491, 528, 515, 509, 34, 19, 0, 0, 0, 0, 0, 0, 504, 528, 503, 534, 486, 527, 507, 454, 29, 13, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4055, 4169, 4127, 4029, 4133, 4135, 4092] avg=4096.0000 max=4169 max_rank=2 min=4028 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[515, 528, 503, 507, 543, 493, 547, 487, 546, 502, 513, 519, 502, 527, 545, 524, 484, 482, 544, 526, 500, 496, 516, 538, 525, 505, 509, 491, 548, 534, 482, 511, 522, 533, 527, 510, 469, 484, 547, 459, 483, 517, 519, 510, 503, 508, 521, 532, 520, 536, 529, 511, 522, 532, 491, 500, 496, 499, 481, 526, 479, 522, 514, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[556, 537, 493, 504, 518, 518, 492, 516, 531, 554, 487, 499, 486, 509, 533, 510, 519, 507, 517, 535, 506, 504, 498, 488, 490, 526, 557, 550, 526, 519, 500, 539, 518, 545, 505, 497, 474, 466, 524, 495, 541, 479, 541, 502, 510, 497, 518, 517, 539, 526, 520, 542, 513, 490, 521, 493, 472, 544, 495, 490, 503, 501, 460, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[552, 526, 514, 491, 553, 506, 462, 489, 505, 528, 505, 520, 492, 482, 492, 519, 524, 520, 541, 515, 515, 504, 500, 493, 520, 524, 533, 530, 498, 523, 474, 490, 502, 525, 536, 514, 541, 526, 538, 516, 492, 539, 503, 518, 524, 492, 520, 522, 516, 534, 511, 474, 511, 514, 503, 499, 494, 508, 520, 481, 522, 517, 514, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[515, 549, 542, 527, 481, 521, 544, 518, 458, 482, 518, 496, 485, 495, 523, 529, 513, 524, 532, 480, 506, 524, 518, 527, 501, 473, 554, 540, 522, 522, 492, 509, 498, 540, 499, 503, 491, 503, 508, 502, 514, 520, 534, 482, 552, 492, 520, 488, 490, 528, 511, 533, 514, 494, 485, 537, 525, 493, 506, 503, 525, 519, 527, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[476, 537, 520, 504, 500, 520, 509, 531, 542, 488, 511, 537, 512, 526, 517, 505, 512, 494, 521, 537, 551, 534, 515, 487, 543, 511, 509, 501, 535, 493, 492, 518, 538, 494, 439, 504, 483, 506, 531, 519, 523, 507, 505, 488, 498, 506, 503, 519, 530, 503, 548, 519, 515, 518, 500, 502, 477, 547, 476, 531, 518, 512, 532, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[494, 528, 546, 500, 529, 507, 530, 504, 518, 560, 543, 508, 506, 498, 508, 518, 561, 516, 486, 516, 480, 521, 508, 514, 516, 504, 503, 544, 484, 492, 516, 543, 474, 521, 529, 491, 511, 529, 526, 539, 525, 477, 521, 528, 508, 544, 500, 501, 487, 510, 499, 513, 511, 487, 537, 520, 450, 498, 509, 502, 508, 508, 480, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[502, 517, 578, 507, 506, 501, 497, 528, 526, 522, 493, 533, 518, 514, 505, 525, 507, 496, 555, 495, 489, 494, 493, 492, 531, 532, 515, 512, 484, 521, 521, 521, 528, 514, 511, 507, 544, 498, 523, 485, 462, 547, 537, 518, 499, 524, 527, 530, 517, 508, 492, 531, 463, 461, 512, 484, 521, 492, 521, 526, 509, 522, 496, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[493, 514, 521, 540, 506, 527, 503, 544, 532, 515, 564, 518, 494, 477, 501, 524, 523, 512, 518, 510, 466, 523, 516, 524, 504, 506, 539, 531, 522, 527, 474, 520, 483, 521, 558, 518, 451, 549, 541, 515, 501, 500, 495, 513, 518, 514, 506, 470, 539, 509, 516, 541, 525, 512, 508, 500, 503, 505, 526, 507, 421, 509, 515, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[552, 489, 514, 491, 553, 506, 462, 489, 0, 0, 0, 0, 0, 0, 0, 0, 494, 528, 505, 520, 492, 482, 492, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 541, 515, 515, 504, 500, 493, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 507, 530, 498, 523, 474, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 536, 514, 541, 526, 538, 516, 5, 3, 0, 0, 0, 0, 0, 0, 492, 539, 503, 518, 524, 492, 520, 522, 5, 0, 0, 0, 0, 0, 0, 0, 516, 534, 511, 474, 511, 514, 503, 499, 1, 0, 0, 0, 0, 0, 0, 0, 494, 508, 520, 481, 522, 517, 514, 502, 37, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4032, 4112, 4066, 4206, 4115, 4063, 4118] avg=4096.0000 max=4206 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 491, 503, 507, 543, 493, 547, 487, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 513, 519, 502, 527, 545, 524, 0, 0, 0, 0, 0, 0, 0, 0, 484, 482, 544, 526, 500, 496, 516, 538, 0, 0, 0, 0, 0, 0, 0, 0, 525, 505, 484, 491, 548, 534, 482, 511, 0, 0, 0, 0, 0, 0, 0, 0, 522, 533, 527, 510, 469, 484, 547, 459, 5, 3, 0, 0, 0, 0, 0, 0, 483, 517, 519, 510, 503, 508, 521, 532, 5, 0, 0, 0, 0, 0, 0, 0, 520, 536, 529, 511, 522, 532, 491, 500, 1, 0, 0, 0, 0, 0, 0, 0, 496, 499, 481, 526, 479, 522, 514, 474, 37, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4167, 4086, 4080, 4059, 4098, 4142, 4050] avg=4096.0000 max=4167 max_rank=1 min=4050 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[556, 500, 493, 504, 518, 518, 492, 516, 0, 0, 0, 0, 0, 0, 0, 0, 520, 554, 487, 499, 486, 509, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 517, 535, 506, 504, 498, 488, 0, 0, 0, 0, 0, 0, 0, 0, 490, 526, 530, 550, 526, 519, 500, 539, 0, 0, 0, 0, 0, 0, 0, 0, 518, 545, 505, 497, 474, 466, 524, 495, 5, 3, 0, 0, 0, 0, 0, 0, 541, 479, 541, 502, 510, 497, 518, 517, 5, 0, 0, 0, 0, 0, 0, 0, 539, 526, 520, 542, 513, 490, 521, 493, 1, 0, 0, 0, 0, 0, 0, 0, 472, 544, 495, 490, 503, 501, 460, 506, 37, 24, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4098, 4074, 4180, 4032, 4110, 4145, 4032] avg=4096.0000 max=4180 max_rank=3 min=4032 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[476, 500, 520, 504, 500, 520, 509, 531, 0, 0, 0, 0, 0, 0, 0, 0, 531, 488, 511, 537, 512, 526, 517, 505, 0, 0, 0, 0, 0, 0, 0, 0, 512, 494, 521, 537, 551, 534, 515, 487, 0, 0, 0, 0, 0, 0, 0, 0, 543, 511, 484, 501, 535, 493, 492, 518, 0, 0, 0, 0, 0, 0, 0, 0, 538, 494, 439, 504, 483, 506, 531, 519, 5, 3, 0, 0, 0, 0, 0, 0, 523, 507, 505, 488, 498, 506, 503, 519, 5, 0, 0, 0, 0, 0, 0, 0, 530, 503, 548, 519, 515, 518, 500, 502, 1, 0, 0, 0, 0, 0, 0, 0, 477, 547, 476, 531, 518, 512, 532, 489, 37, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4060, 4127, 4151, 4077, 4022, 4054, 4136, 4141] avg=4096.0000 max=4151 max_rank=2 min=4022 min_rank=4 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 511, 542, 527, 481, 521, 544, 518, 0, 0, 0, 0, 0, 0, 0, 0, 449, 482, 518, 496, 485, 495, 523, 529, 0, 0, 0, 0, 0, 0, 0, 0, 513, 524, 532, 480, 506, 524, 518, 527, 0, 0, 0, 0, 0, 0, 0, 0, 501, 473, 527, 540, 522, 522, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 498, 540, 499, 503, 491, 503, 508, 502, 4, 3, 0, 0, 0, 0, 0, 0, 514, 520, 534, 482, 552, 492, 520, 488, 4, 0, 0, 0, 0, 0, 0, 0, 490, 528, 511, 533, 514, 494, 485, 537, 1, 0, 0, 0, 0, 0, 0, 0, 525, 493, 506, 503, 525, 519, 527, 512, 38, 24, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 3977, 4124, 4086, 4051, 4106, 4093, 4172] avg=4096.0000 max=4172 max_rank=7 min=3977 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[502, 481, 578, 507, 506, 501, 497, 528, 0, 0, 0, 0, 0, 0, 0, 0, 515, 522, 493, 533, 518, 514, 505, 525, 0, 0, 0, 0, 0, 0, 0, 0, 507, 496, 555, 495, 489, 494, 493, 492, 0, 0, 0, 0, 0, 0, 0, 0, 531, 532, 490, 512, 484, 521, 521, 521, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 511, 507, 544, 498, 523, 485, 5, 3, 0, 0, 0, 0, 0, 0, 462, 547, 537, 518, 499, 524, 527, 530, 5, 0, 0, 0, 0, 0, 0, 0, 517, 508, 492, 531, 463, 461, 512, 484, 1, 0, 0, 0, 0, 0, 0, 0, 521, 492, 521, 526, 509, 522, 496, 529, 36, 22, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4125, 4021, 4112, 4118, 4149, 3969, 4174] avg=4096.0000 max=4174 max_rank=7 min=3969 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[494, 488, 546, 500, 529, 507, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 487, 560, 543, 508, 506, 498, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 561, 516, 486, 516, 480, 521, 508, 514, 6, 0, 0, 0, 0, 0, 0, 0, 516, 504, 471, 544, 484, 492, 516, 543, 0, 0, 0, 0, 0, 0, 0, 0, 474, 521, 529, 491, 511, 529, 526, 539, 10, 6, 0, 0, 0, 0, 0, 0, 525, 477, 521, 528, 508, 544, 500, 501, 10, 0, 0, 0, 0, 0, 0, 0, 487, 510, 499, 513, 511, 487, 537, 520, 5, 0, 0, 0, 0, 0, 0, 0, 450, 498, 509, 502, 508, 508, 480, 524, 40, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4128, 4108, 4070, 4136, 4114, 4069, 4045] avg=4096.0000 max=4136 max_rank=4 min=4045 min_rank=7 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 478, 521, 540, 506, 527, 503, 544, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 564, 518, 494, 477, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 523, 512, 518, 510, 466, 523, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 513, 531, 522, 527, 474, 520, 0, 0, 0, 0, 0, 0, 0, 0, 483, 521, 558, 518, 451, 549, 541, 515, 5, 3, 0, 0, 0, 0, 0, 0, 501, 500, 495, 513, 518, 514, 506, 470, 5, 0, 0, 0, 0, 0, 0, 0, 539, 509, 516, 541, 525, 512, 508, 500, 1, 0, 0, 0, 0, 0, 0, 0, 503, 505, 526, 507, 421, 509, 515, 491, 36, 23, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4114, 4092, 4097, 4144, 4022, 4151, 4036] avg=4096.0000 max=4151 max_rank=6 min=4022 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[509, 519, 510, 502, 532, 490, 513, 523, 491, 515, 516, 530, 503, 536, 506, 522, 513, 519, 500, 490, 507, 568, 491, 532, 519, 532, 528, 507, 520, 503, 488, 520, 505, 523, 536, 529, 494, 545, 497, 540, 519, 531, 510, 489, 525, 537, 495, 493, 519, 505, 455, 545, 519, 477, 497, 508, 525, 492, 475, 498, 508, 519, 500, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[489, 550, 517, 558, 520, 524, 538, 534, 492, 542, 499, 492, 557, 495, 484, 528, 526, 531, 490, 482, 547, 498, 512, 522, 504, 505, 516, 545, 521, 518, 486, 537, 486, 512, 531, 494, 493, 527, 474, 489, 502, 538, 521, 524, 540, 508, 504, 527, 493, 512, 523, 505, 505, 511, 462, 508, 480, 503, 537, 463, 496, 524, 507, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[501, 533, 521, 529, 529, 492, 522, 549, 531, 533, 509, 548, 500, 501, 523, 501, 514, 506, 523, 528, 500, 510, 528, 498, 523, 489, 545, 481, 534, 505, 516, 480, 505, 495, 500, 502, 511, 545, 502, 551, 543, 491, 521, 504, 508, 507, 483, 503, 514, 503, 519, 486, 541, 528, 532, 466, 509, 511, 511, 487, 525, 487, 488, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[502, 537, 536, 527, 556, 552, 521, 500, 505, 506, 521, 549, 524, 525, 498, 520, 538, 508, 527, 492, 494, 549, 503, 521, 518, 530, 508, 481, 506, 513, 500, 502, 505, 472, 511, 528, 514, 535, 528, 517, 505, 473, 478, 456, 483, 516, 510, 501, 533, 533, 482, 474, 513, 534, 493, 498, 505, 507, 534, 522, 484, 498, 546, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[522, 519, 525, 536, 546, 513, 494, 512, 500, 543, 500, 484, 508, 549, 559, 487, 514, 474, 516, 526, 508, 512, 516, 505, 496, 538, 529, 559, 509, 537, 499, 524, 505, 491, 518, 500, 504, 520, 501, 501, 513, 504, 551, 449, 532, 497, 502, 538, 501, 495, 522, 486, 490, 526, 513, 530, 464, 508, 515, 498, 494, 517, 513, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[544, 499, 457, 506, 521, 546, 507, 547, 520, 517, 490, 545, 516, 499, 515, 501, 546, 531, 490, 509, 487, 519, 517, 462, 528, 518, 523, 567, 530, 468, 534, 508, 497, 523, 488, 496, 513, 491, 568, 533, 496, 516, 507, 513, 484, 511, 492, 531, 493, 512, 515, 524, 537, 505, 492, 513, 495, 494, 525, 517, 502, 516, 478, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[528, 505, 509, 559, 489, 510, 494, 545, 511, 513, 506, 497, 495, 490, 484, 502, 492, 532, 540, 515, 482, 528, 484, 495, 551, 481, 522, 560, 467, 510, 536, 534, 519, 498, 503, 519, 518, 526, 520, 530, 553, 497, 515, 533, 539, 500, 551, 494, 514, 479, 530, 491, 511, 525, 500, 496, 521, 533, 518, 514, 508, 497, 463, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[519, 525, 472, 523, 487, 526, 523, 492, 479, 504, 520, 512, 506, 497, 544, 496, 501, 515, 501, 518, 479, 481, 530, 493, 519, 542, 509, 521, 497, 507, 525, 519, 504, 518, 500, 537, 568, 503, 524, 511, 510, 512, 516, 464, 512, 522, 557, 517, 530, 484, 516, 505, 488, 493, 537, 559, 515, 468, 512, 531, 538, 505, 476, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[502, 537, 536, 470, 556, 552, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 505, 504, 521, 549, 524, 525, 498, 520, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 527, 492, 494, 549, 503, 521, 7, 2, 0, 0, 0, 0, 0, 0, 518, 530, 508, 450, 506, 513, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 505, 472, 511, 528, 514, 527, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 505, 473, 478, 456, 483, 516, 510, 501, 9, 1, 0, 0, 0, 0, 0, 0, 533, 533, 482, 474, 513, 534, 493, 498, 22, 7, 0, 0, 0, 0, 0, 0, 505, 507, 534, 522, 484, 498, 546, 511, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4146, 4141, 4027, 4102, 3932, 4089, 4157] avg=4096.0000 max=4174 max_rank=0 min=3932 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 519, 510, 448, 532, 490, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 491, 513, 516, 530, 503, 536, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 519, 500, 490, 507, 568, 491, 532, 7, 2, 0, 0, 0, 0, 0, 0, 519, 532, 528, 475, 520, 503, 488, 520, 0, 0, 0, 0, 0, 0, 0, 0, 505, 523, 536, 529, 494, 537, 497, 540, 0, 0, 0, 0, 0, 0, 0, 0, 519, 531, 510, 489, 525, 537, 495, 493, 9, 1, 0, 0, 0, 0, 0, 0, 519, 505, 455, 545, 519, 477, 497, 508, 23, 7, 0, 0, 0, 0, 0, 0, 525, 492, 475, 498, 508, 519, 500, 504, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4117, 4129, 4085, 4161, 4109, 4055, 4068] avg=4096.0000 max=4161 max_rank=4 min=4044 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[489, 550, 517, 497, 520, 524, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 492, 540, 499, 492, 557, 495, 484, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 531, 490, 482, 547, 498, 512, 522, 7, 2, 0, 0, 0, 0, 0, 0, 504, 505, 516, 510, 521, 518, 486, 537, 0, 0, 0, 0, 0, 0, 0, 0, 486, 512, 531, 494, 493, 519, 474, 489, 0, 0, 0, 0, 0, 0, 0, 0, 502, 538, 521, 524, 540, 508, 504, 527, 10, 1, 0, 0, 0, 0, 0, 0, 493, 512, 523, 505, 505, 511, 462, 508, 25, 8, 0, 0, 0, 0, 0, 0, 480, 503, 537, 463, 496, 524, 507, 510, 53, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4087, 4117, 4097, 3998, 4175, 4052, 4073] avg=4096.0000 max=4175 max_rank=5 min=3998 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[501, 533, 521, 472, 529, 492, 522, 549, 0, 0, 0, 0, 0, 0, 0, 0, 531, 531, 509, 548, 500, 501, 523, 501, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 523, 528, 500, 510, 528, 498, 7, 2, 0, 0, 0, 0, 0, 0, 523, 489, 545, 450, 534, 505, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 505, 495, 500, 502, 511, 537, 502, 551, 0, 0, 0, 0, 0, 0, 0, 0, 543, 491, 521, 504, 508, 507, 483, 503, 9, 1, 0, 0, 0, 0, 0, 0, 514, 503, 519, 486, 541, 528, 532, 466, 22, 7, 0, 0, 0, 0, 0, 0, 509, 511, 511, 487, 525, 487, 488, 488, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4144, 4116, 4042, 4103, 4070, 4118, 4056] avg=4096.0000 max=4144 max_rank=1 min=4042 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 519, 525, 478, 546, 513, 494, 512, 0, 0, 0, 0, 0, 0, 0, 0, 500, 541, 500, 484, 508, 549, 559, 487, 0, 0, 0, 0, 0, 0, 0, 0, 514, 474, 516, 526, 508, 512, 516, 505, 7, 2, 0, 0, 0, 0, 0, 0, 496, 538, 529, 523, 509, 537, 499, 524, 0, 0, 0, 0, 0, 0, 0, 0, 505, 491, 518, 500, 504, 512, 501, 501, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 551, 449, 532, 497, 502, 538, 10, 1, 0, 0, 0, 0, 0, 0, 501, 495, 522, 486, 490, 526, 513, 530, 26, 8, 0, 0, 0, 0, 0, 0, 464, 508, 515, 498, 494, 517, 513, 511, 50, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4128, 4080, 4155, 4032, 4097, 4097, 4070] avg=4096.0000 max=4155 max_rank=3 min=4032 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 505, 509, 498, 489, 510, 494, 545, 0, 0, 0, 0, 0, 0, 0, 0, 511, 511, 506, 497, 495, 490, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 532, 540, 515, 482, 528, 484, 495, 7, 2, 0, 0, 0, 0, 0, 0, 551, 481, 522, 524, 467, 510, 536, 534, 0, 0, 0, 0, 0, 0, 0, 0, 519, 498, 503, 519, 518, 518, 520, 530, 0, 0, 0, 0, 0, 0, 0, 0, 553, 497, 515, 533, 539, 500, 551, 494, 10, 1, 0, 0, 0, 0, 0, 0, 514, 479, 530, 491, 511, 525, 500, 496, 26, 8, 0, 0, 0, 0, 0, 0, 521, 533, 518, 514, 508, 497, 463, 487, 53, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3996, 4077, 4125, 4125, 4193, 4080, 4094] avg=4096.0000 max=4193 max_rank=5 min=3996 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[544, 499, 457, 443, 521, 546, 507, 547, 0, 0, 0, 0, 0, 0, 0, 0, 520, 508, 490, 545, 516, 499, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 546, 531, 490, 509, 487, 519, 517, 462, 11, 9, 0, 0, 0, 0, 0, 0, 528, 518, 523, 522, 530, 468, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 523, 488, 496, 513, 472, 568, 533, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 507, 513, 484, 511, 492, 531, 14, 8, 0, 0, 0, 0, 0, 0, 493, 512, 515, 524, 537, 505, 492, 513, 31, 12, 0, 0, 0, 0, 0, 0, 495, 494, 525, 517, 502, 516, 478, 524, 51, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4094, 4081, 4131, 4090, 4072, 4134, 4102] avg=4096.0000 max=4134 max_rank=6 min=4064 min_rank=0 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16398 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 525, 472, 467, 487, 526, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 479, 502, 520, 512, 506, 497, 544, 496, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 501, 518, 479, 481, 530, 493, 7, 2, 0, 0, 0, 0, 0, 0, 519, 542, 509, 488, 497, 507, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 500, 537, 568, 495, 524, 511, 0, 0, 0, 0, 0, 0, 0, 0, 510, 512, 516, 464, 512, 522, 557, 517, 9, 1, 0, 0, 0, 0, 0, 0, 530, 484, 516, 505, 488, 493, 537, 559, 24, 7, 0, 0, 0, 0, 0, 0, 515, 468, 512, 531, 538, 505, 476, 554, 49, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4011, 4056, 4027, 4106, 4157, 4120, 4143, 4148] avg=4096.0000 max=4157 max_rank=4 min=4011 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 512, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 3968, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3840, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[505, 519, 534, 457, 492, 498, 526, 510, 544, 517, 516, 480, 529, 516, 493, 501, 534, 477, 502, 510, 510, 492, 512, 516, 528, 547, 494, 502, 511, 516, 519, 534, 516, 493, 510, 532, 524, 496, 502, 509, 527, 512, 522, 538, 537, 535, 506, 554, 497, 465, 531, 551, 501, 482, 506, 552, 508, 524, 498, 536, 469, 516, 490, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[520, 506, 487, 501, 505, 540, 526, 518, 552, 511, 527, 520, 514, 530, 516, 542, 530, 525, 515, 523, 499, 543, 513, 499, 493, 490, 517, 493, 509, 556, 514, 505, 508, 479, 500, 493, 508, 514, 531, 531, 481, 509, 531, 494, 500, 460, 475, 523, 519, 491, 475, 545, 513, 506, 506, 474, 549, 494, 521, 518, 507, 527, 517, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[492, 499, 522, 511, 516, 502, 530, 501, 488, 529, 486, 551, 520, 536, 520, 522, 505, 511, 506, 483, 542, 495, 520, 495, 527, 531, 531, 500, 531, 519, 487, 510, 552, 507, 521, 526, 475, 499, 528, 502, 493, 498, 552, 518, 506, 468, 505, 540, 506, 488, 533, 504, 504, 552, 516, 507, 492, 533, 489, 531, 521, 513, 472, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[511, 539, 523, 515, 523, 542, 520, 527, 487, 497, 529, 537, 509, 488, 501, 536, 540, 523, 526, 552, 467, 517, 477, 490, 507, 510, 492, 487, 498, 474, 535, 516, 474, 534, 557, 500, 519, 520, 561, 541, 467, 479, 527, 505, 478, 505, 509, 562, 552, 451, 537, 499, 467, 557, 501, 481, 488, 532, 530, 495, 490, 535, 485, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[528, 501, 518, 535, 477, 499, 478, 521, 519, 546, 520, 514, 476, 494, 496, 489, 508, 465, 513, 516, 522, 524, 481, 475, 554, 515, 517, 505, 516, 556, 490, 514, 506, 537, 553, 512, 529, 554, 490, 490, 517, 508, 547, 497, 524, 517, 541, 549, 532, 514, 497, 491, 465, 509, 489, 520, 493, 477, 534, 553, 491, 513, 517, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[481, 511, 502, 526, 507, 530, 512, 490, 501, 508, 517, 499, 515, 547, 532, 526, 489, 518, 537, 550, 574, 510, 526, 508, 514, 516, 503, 538, 526, 499, 522, 499, 495, 504, 536, 500, 495, 467, 530, 527, 504, 494, 520, 540, 476, 517, 512, 503, 509, 529, 495, 527, 499, 451, 521, 494, 515, 497, 498, 528, 473, 503, 550, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[516, 514, 539, 550, 538, 528, 534, 543, 522, 501, 548, 505, 516, 501, 480, 538, 541, 461, 511, 508, 524, 556, 476, 515, 486, 503, 499, 481, 508, 518, 514, 529, 528, 496, 512, 507, 503, 501, 527, 512, 485, 490, 520, 506, 540, 535, 476, 499, 457, 504, 513, 483, 536, 523, 491, 515, 564, 538, 489, 512, 494, 520, 485, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[521, 517, 516, 516, 539, 492, 511, 526, 537, 488, 513, 471, 520, 515, 512, 480, 500, 470, 509, 524, 487, 520, 520, 568, 564, 573, 517, 523, 514, 499, 477, 509, 520, 507, 536, 510, 529, 511, 534, 501, 483, 522, 490, 511, 516, 511, 517, 504, 523, 489, 523, 524, 490, 474, 505, 554, 479, 534, 526, 519, 523, 491, 490, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 519, 513, 457, 492, 498, 526, 510, 0, 0, 0, 0, 0, 0, 0, 0, 544, 517, 502, 480, 529, 516, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 534, 477, 502, 510, 510, 492, 512, 516, 1, 0, 0, 0, 0, 0, 0, 0, 528, 534, 494, 502, 511, 516, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 492, 532, 524, 496, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 527, 512, 522, 538, 537, 535, 506, 554, 10, 0, 0, 0, 0, 0, 0, 0, 497, 465, 531, 551, 501, 482, 506, 552, 21, 18, 4, 0, 0, 0, 0, 0, 508, 524, 498, 536, 469, 516, 490, 488, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4082, 4054, 4138, 4064, 4241, 4128, 4041] avg=4096.0000 max=4241 max_rank=5 min=4020 min_rank=0 imbalance_ratio=1.035400 max_load_reduction=28527 max_load_reduction_pct=0.870575 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 539, 503, 515, 523, 542, 520, 527, 0, 0, 0, 0, 0, 0, 0, 0, 487, 497, 514, 537, 509, 488, 501, 536, 0, 0, 0, 0, 0, 0, 0, 0, 540, 523, 526, 552, 467, 517, 477, 490, 1, 0, 0, 0, 0, 0, 0, 0, 507, 498, 492, 487, 498, 474, 535, 516, 0, 0, 0, 0, 0, 0, 0, 0, 474, 534, 537, 500, 519, 520, 561, 541, 0, 0, 0, 0, 0, 0, 0, 0, 467, 479, 527, 505, 478, 505, 509, 562, 9, 0, 0, 0, 0, 0, 0, 0, 552, 451, 537, 499, 467, 557, 501, 481, 20, 20, 4, 0, 0, 0, 0, 0, 488, 532, 530, 495, 490, 535, 485, 535, 11, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4069, 4093, 4007, 4186, 4041, 4089, 4103] avg=4096.0000 max=4186 max_rank=4 min=4007 min_rank=3 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 506, 468, 501, 505, 540, 526, 518, 0, 0, 0, 0, 0, 0, 0, 0, 552, 511, 512, 520, 514, 530, 516, 542, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 515, 523, 499, 543, 513, 499, 1, 0, 0, 0, 0, 0, 0, 0, 493, 478, 517, 493, 509, 556, 514, 505, 0, 0, 0, 0, 0, 0, 0, 0, 508, 479, 482, 493, 508, 514, 531, 531, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 531, 494, 500, 460, 475, 523, 9, 0, 0, 0, 0, 0, 0, 0, 519, 491, 475, 545, 513, 506, 506, 474, 19, 18, 4, 0, 0, 0, 0, 0, 549, 494, 521, 518, 507, 527, 517, 530, 11, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4197, 4148, 4065, 4046, 3982, 4070, 4176] avg=4096.0000 max=4197 max_rank=1 min=3982 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[528, 501, 498, 535, 477, 499, 478, 521, 0, 0, 0, 0, 0, 0, 0, 0, 519, 546, 506, 514, 476, 494, 496, 489, 0, 0, 0, 0, 0, 0, 0, 0, 508, 465, 513, 516, 522, 524, 481, 475, 1, 0, 0, 0, 0, 0, 0, 0, 554, 503, 517, 505, 516, 556, 490, 514, 0, 0, 0, 0, 0, 0, 0, 0, 506, 537, 533, 512, 529, 554, 490, 490, 0, 0, 0, 0, 0, 0, 0, 0, 517, 508, 547, 497, 524, 517, 541, 549, 9, 0, 0, 0, 0, 0, 0, 0, 532, 514, 497, 491, 465, 509, 489, 520, 20, 20, 4, 0, 0, 0, 0, 0, 493, 477, 534, 553, 491, 513, 517, 520, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4040, 4005, 4155, 4151, 4209, 4061, 4110] avg=4096.0000 max=4209 max_rank=5 min=4005 min_rank=2 imbalance_ratio=1.027588 max_load_reduction=28559 max_load_reduction_pct=0.871552 moved_to_echo=16531 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[492, 499, 502, 511, 516, 502, 530, 501, 0, 0, 0, 0, 0, 0, 0, 0, 488, 529, 472, 551, 520, 536, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 506, 483, 542, 495, 520, 495, 1, 0, 0, 0, 0, 0, 0, 0, 527, 518, 531, 500, 531, 519, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 552, 507, 503, 526, 475, 499, 528, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 552, 518, 506, 468, 505, 540, 10, 0, 0, 0, 0, 0, 0, 0, 506, 488, 533, 504, 504, 552, 516, 507, 20, 18, 4, 0, 0, 0, 0, 0, 492, 533, 489, 531, 521, 513, 472, 499, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4138, 4058, 4123, 4092, 4090, 4152, 4062] avg=4096.0000 max=4152 max_rank=6 min=4053 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[521, 517, 496, 516, 539, 492, 511, 526, 0, 0, 0, 0, 0, 0, 0, 0, 537, 488, 499, 471, 520, 515, 512, 480, 0, 0, 0, 0, 0, 0, 0, 0, 500, 470, 509, 524, 487, 520, 520, 568, 1, 0, 0, 0, 0, 0, 0, 0, 564, 559, 517, 523, 514, 499, 477, 509, 0, 0, 0, 0, 0, 0, 0, 0, 520, 507, 517, 510, 529, 511, 534, 501, 0, 0, 0, 0, 0, 0, 0, 0, 483, 522, 490, 511, 516, 511, 517, 504, 11, 0, 0, 0, 0, 0, 0, 0, 523, 489, 523, 524, 490, 474, 505, 554, 20, 19, 4, 0, 0, 0, 0, 0, 479, 534, 526, 519, 523, 491, 490, 474, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4022, 4099, 4162, 4129, 4065, 4125, 4048] avg=4096.0000 max=4162 max_rank=3 min=4022 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 514, 514, 550, 538, 528, 534, 543, 0, 0, 0, 0, 0, 0, 0, 0, 522, 501, 526, 505, 516, 501, 480, 538, 0, 0, 0, 0, 0, 0, 0, 0, 541, 461, 511, 508, 524, 556, 476, 515, 6, 0, 0, 0, 0, 0, 0, 0, 486, 478, 499, 481, 508, 518, 514, 529, 0, 0, 0, 0, 0, 0, 0, 0, 528, 496, 491, 507, 503, 501, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 485, 490, 520, 506, 540, 535, 476, 499, 14, 0, 0, 0, 0, 0, 0, 0, 457, 504, 513, 483, 536, 523, 491, 515, 25, 21, 7, 0, 0, 0, 0, 0, 564, 538, 489, 512, 494, 520, 485, 504, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4237, 4089, 4098, 4013, 4065, 4065, 4075, 4126] avg=4096.0000 max=4237 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 511, 482, 526, 507, 530, 512, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 503, 499, 515, 547, 532, 526, 0, 0, 0, 0, 0, 0, 0, 0, 489, 518, 537, 550, 574, 510, 526, 508, 1, 0, 0, 0, 0, 0, 0, 0, 514, 504, 503, 538, 526, 499, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 517, 500, 495, 467, 530, 527, 0, 0, 0, 0, 0, 0, 0, 0, 504, 494, 520, 540, 476, 517, 512, 503, 9, 0, 0, 0, 0, 0, 0, 0, 509, 529, 495, 527, 499, 451, 521, 494, 20, 19, 4, 0, 0, 0, 0, 0, 515, 497, 498, 528, 473, 503, 550, 526, 10, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4131, 4213, 4105, 4035, 4075, 4068, 4102] avg=4096.0000 max=4213 max_rank=2 min=4035 min_rank=4 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4096, 4352, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[565, 497, 533, 504, 507, 523, 528, 498, 510, 510, 507, 541, 506, 523, 492, 526, 531, 525, 530, 484, 507, 552, 536, 517, 514, 513, 491, 501, 488, 500, 538, 498, 508, 513, 515, 493, 473, 495, 493, 508, 492, 501, 524, 532, 514, 502, 492, 502, 506, 534, 515, 547, 506, 505, 520, 502, 504, 501, 504, 510, 505, 483, 541, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[483, 551, 537, 567, 484, 521, 520, 539, 492, 524, 497, 487, 509, 526, 512, 553, 494, 531, 495, 494, 517, 501, 553, 525, 536, 523, 493, 519, 512, 502, 512, 517, 523, 555, 498, 484, 490, 498, 491, 521, 491, 531, 531, 527, 506, 494, 491, 547, 507, 466, 497, 528, 500, 528, 521, 501, 524, 496, 490, 508, 509, 529, 469, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[525, 528, 497, 532, 516, 540, 521, 474, 541, 530, 534, 498, 521, 481, 539, 505, 503, 505, 512, 517, 510, 527, 529, 574, 509, 514, 538, 478, 528, 523, 512, 496, 460, 499, 495, 481, 510, 498, 477, 496, 541, 513, 510, 524, 533, 499, 477, 499, 516, 540, 515, 500, 498, 530, 524, 513, 511, 531, 484, 489, 498, 520, 509, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[484, 514, 495, 505, 481, 511, 464, 499, 546, 572, 535, 501, 509, 537, 535, 510, 517, 540, 528, 549, 528, 532, 535, 509, 533, 506, 529, 511, 539, 478, 480, 512, 545, 536, 509, 510, 486, 512, 501, 505, 523, 518, 498, 489, 489, 472, 491, 525, 507, 512, 556, 507, 525, 502, 490, 506, 496, 504, 500, 484, 480, 510, 546, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[474, 527, 531, 495, 533, 459, 560, 496, 522, 536, 485, 504, 541, 522, 505, 494, 504, 530, 509, 535, 516, 516, 468, 513, 505, 540, 518, 491, 520, 501, 521, 531, 515, 499, 525, 493, 507, 532, 537, 513, 493, 521, 525, 534, 514, 495, 510, 475, 483, 537, 534, 518, 543, 501, 497, 489, 488, 545, 516, 533, 512, 491, 512, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[551, 506, 518, 486, 514, 500, 546, 542, 516, 536, 517, 521, 536, 518, 521, 469, 540, 530, 493, 516, 498, 531, 546, 484, 544, 494, 496, 520, 505, 509, 481, 481, 496, 479, 581, 514, 494, 488, 523, 515, 473, 541, 490, 532, 536, 508, 515, 455, 483, 504, 471, 516, 531, 542, 508, 518, 511, 501, 508, 496, 537, 507, 512, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[518, 490, 508, 504, 537, 488, 525, 520, 532, 482, 526, 496, 523, 513, 479, 493, 478, 539, 536, 517, 541, 494, 544, 544, 537, 490, 563, 527, 551, 494, 507, 499, 489, 511, 530, 488, 483, 513, 529, 512, 541, 531, 507, 527, 492, 513, 502, 527, 510, 498, 544, 488, 511, 530, 474, 501, 510, 480, 510, 497, 515, 535, 504, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[527, 522, 517, 498, 511, 576, 479, 543, 472, 534, 487, 497, 464, 523, 509, 547, 475, 507, 528, 502, 532, 472, 486, 502, 529, 480, 516, 491, 534, 530, 501, 570, 495, 521, 524, 509, 487, 490, 531, 513, 499, 508, 501, 529, 495, 513, 522, 541, 533, 520, 505, 505, 495, 492, 538, 532, 504, 516, 541, 510, 518, 541, 471, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 495, 505, 481, 511, 445, 499, 0, 0, 0, 0, 0, 0, 0, 0, 546, 551, 535, 501, 509, 537, 535, 510, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 528, 549, 528, 532, 535, 509, 0, 0, 0, 0, 0, 0, 0, 0, 527, 506, 529, 511, 539, 478, 480, 512, 0, 0, 0, 0, 0, 0, 0, 0, 545, 536, 509, 510, 486, 512, 501, 505, 45, 0, 0, 0, 0, 0, 0, 0, 523, 518, 498, 489, 489, 472, 491, 525, 9, 6, 0, 0, 0, 0, 0, 0, 507, 512, 556, 507, 525, 502, 490, 506, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 500, 484, 480, 510, 546, 510, 19, 12, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3934, 4224, 4192, 4082, 4149, 4020, 4105, 4062] avg=4096.0000 max=4224 max_rank=1 min=3934 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[483, 551, 537, 567, 484, 521, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 497, 487, 509, 526, 512, 553, 0, 0, 0, 0, 0, 0, 0, 0, 494, 486, 495, 494, 517, 501, 553, 525, 0, 0, 0, 0, 0, 0, 0, 0, 530, 523, 493, 519, 512, 502, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 523, 555, 498, 484, 490, 498, 491, 521, 44, 0, 0, 0, 0, 0, 0, 0, 491, 531, 531, 527, 506, 494, 491, 547, 8, 6, 0, 0, 0, 0, 0, 0, 507, 466, 497, 528, 500, 528, 521, 501, 0, 0, 0, 0, 0, 0, 0, 0, 524, 496, 490, 508, 509, 529, 469, 491, 22, 11, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4180, 4081, 4065, 4108, 4104, 4132, 4048, 4050] avg=4096.0000 max=4180 max_rank=0 min=4048 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 528, 497, 532, 516, 540, 499, 474, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 534, 498, 521, 481, 539, 505, 0, 0, 0, 0, 0, 0, 0, 0, 503, 462, 512, 517, 510, 527, 529, 574, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 538, 478, 528, 523, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 460, 499, 495, 481, 510, 498, 477, 496, 42, 0, 0, 0, 0, 0, 0, 0, 541, 513, 510, 524, 533, 499, 477, 499, 8, 6, 0, 0, 0, 0, 0, 0, 516, 540, 515, 500, 498, 530, 524, 513, 0, 0, 0, 0, 0, 0, 0, 0, 511, 531, 484, 489, 498, 520, 509, 521, 22, 11, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4130, 4134, 4092, 3958, 4110, 4136, 4097] avg=4096.0000 max=4136 max_rank=6 min=3958 min_rank=4 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16301 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[565, 497, 533, 504, 507, 523, 506, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 507, 541, 506, 523, 492, 526, 0, 0, 0, 0, 0, 0, 0, 0, 531, 480, 530, 484, 507, 552, 536, 517, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 491, 501, 488, 500, 538, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 515, 493, 473, 495, 493, 508, 44, 0, 0, 0, 0, 0, 0, 0, 492, 501, 524, 532, 514, 502, 492, 502, 8, 6, 0, 0, 0, 0, 0, 0, 506, 534, 515, 547, 506, 505, 520, 502, 0, 0, 0, 0, 0, 0, 0, 0, 504, 501, 504, 510, 505, 483, 541, 533, 22, 11, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4096, 4137, 4037, 4042, 4073, 4135, 4115] avg=4096.0000 max=4137 max_rank=2 min=4037 min_rank=3 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16365 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[474, 527, 531, 495, 533, 459, 537, 496, 0, 0, 0, 0, 0, 0, 0, 0, 522, 517, 485, 504, 541, 522, 505, 494, 0, 0, 0, 0, 0, 0, 0, 0, 504, 485, 509, 535, 516, 516, 468, 513, 0, 0, 0, 0, 0, 0, 0, 0, 499, 540, 518, 491, 520, 501, 521, 531, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 525, 493, 507, 532, 537, 513, 44, 0, 0, 0, 0, 0, 0, 0, 493, 521, 525, 534, 514, 495, 510, 475, 8, 6, 0, 0, 0, 0, 0, 0, 483, 537, 534, 518, 543, 501, 497, 489, 0, 0, 0, 0, 0, 0, 0, 0, 488, 545, 516, 533, 512, 491, 512, 479, 23, 11, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4090, 4046, 4121, 4165, 4081, 4102, 4111] avg=4096.0000 max=4165 max_rank=4 min=4046 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[527, 522, 517, 498, 511, 576, 459, 543, 0, 0, 0, 0, 0, 0, 0, 0, 472, 515, 487, 497, 464, 523, 509, 547, 0, 0, 0, 0, 0, 0, 0, 0, 475, 464, 528, 502, 532, 472, 486, 502, 0, 0, 0, 0, 0, 0, 0, 0, 523, 480, 516, 491, 534, 530, 501, 570, 0, 0, 0, 0, 0, 0, 0, 0, 495, 521, 524, 509, 487, 490, 531, 513, 42, 0, 0, 0, 0, 0, 0, 0, 499, 508, 501, 529, 495, 513, 522, 541, 8, 6, 0, 0, 0, 0, 0, 0, 533, 520, 505, 505, 495, 492, 538, 532, 0, 0, 0, 0, 0, 0, 0, 0, 504, 516, 541, 510, 518, 541, 471, 508, 20, 11, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4014, 3961, 4145, 4112, 4122, 4120, 4141] avg=4096.0000 max=4153 max_rank=0 min=3961 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[518, 490, 508, 504, 537, 488, 503, 520, 0, 0, 0, 0, 0, 0, 0, 0, 532, 465, 526, 496, 523, 513, 479, 493, 0, 0, 0, 0, 0, 0, 0, 0, 478, 493, 536, 517, 541, 494, 544, 544, 0, 0, 0, 0, 0, 0, 0, 0, 531, 490, 563, 527, 551, 494, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 489, 511, 530, 488, 483, 513, 529, 512, 45, 0, 0, 0, 0, 0, 0, 0, 541, 531, 507, 527, 492, 513, 502, 527, 7, 6, 0, 0, 0, 0, 0, 0, 510, 498, 544, 488, 511, 530, 474, 501, 0, 0, 0, 0, 0, 0, 0, 0, 510, 480, 510, 497, 515, 535, 504, 471, 22, 10, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4027, 4147, 4162, 4100, 4153, 4056, 4055] avg=4096.0000 max=4162 max_rank=3 min=4027 min_rank=1 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[551, 506, 518, 486, 514, 500, 520, 542, 0, 0, 0, 0, 0, 0, 0, 0, 516, 508, 517, 521, 536, 518, 521, 469, 0, 0, 0, 0, 0, 0, 0, 0, 540, 478, 493, 516, 498, 531, 546, 484, 0, 0, 0, 0, 0, 0, 0, 0, 535, 494, 496, 520, 505, 509, 481, 481, 0, 0, 0, 0, 0, 0, 0, 0, 496, 479, 581, 514, 494, 488, 523, 515, 48, 0, 0, 0, 0, 0, 0, 0, 473, 541, 490, 532, 536, 508, 515, 455, 11, 9, 7, 0, 0, 0, 0, 0, 483, 504, 464, 516, 531, 542, 508, 518, 0, 0, 0, 0, 0, 0, 0, 0, 511, 501, 508, 496, 537, 507, 512, 518, 26, 17, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4106, 4086, 4021, 4138, 4077, 4066, 4137] avg=4096.0000 max=4138 max_rank=4 min=4021 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 3968, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[510, 500, 517, 502, 535, 532, 527, 485, 485, 533, 513, 510, 486, 497, 543, 525, 538, 508, 544, 479, 513, 478, 509, 566, 497, 514, 515, 522, 484, 503, 508, 550, 527, 518, 493, 524, 533, 544, 490, 491, 498, 526, 519, 532, 515, 511, 502, 473, 503, 515, 484, 504, 503, 565, 515, 487, 494, 484, 533, 541, 505, 502, 499, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[537, 506, 535, 517, 508, 503, 513, 494, 506, 492, 526, 487, 527, 522, 523, 486, 540, 553, 472, 503, 507, 521, 475, 496, 501, 532, 519, 542, 519, 510, 515, 557, 516, 524, 497, 513, 473, 499, 514, 519, 508, 523, 536, 513, 497, 485, 557, 526, 532, 522, 532, 506, 526, 515, 521, 443, 498, 499, 520, 487, 486, 521, 501, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[470, 517, 490, 538, 574, 513, 503, 505, 508, 545, 520, 498, 492, 517, 498, 522, 464, 495, 506, 539, 540, 482, 504, 537, 496, 503, 507, 500, 540, 525, 497, 518, 539, 509, 493, 524, 525, 479, 494, 499, 517, 514, 530, 490, 510, 511, 488, 537, 487, 519, 516, 470, 515, 529, 502, 541, 526, 533, 473, 534, 488, 525, 574, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[490, 515, 479, 515, 535, 527, 512, 512, 538, 517, 505, 573, 460, 566, 522, 533, 500, 528, 513, 563, 532, 528, 506, 504, 511, 507, 519, 503, 522, 531, 540, 500, 517, 503, 531, 510, 473, 504, 497, 534, 468, 487, 482, 518, 522, 492, 514, 494, 519, 491, 503, 515, 495, 496, 539, 516, 512, 522, 513, 496, 506, 496, 485, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[479, 482, 507, 542, 542, 517, 525, 504, 515, 473, 536, 532, 529, 494, 517, 512, 467, 513, 487, 518, 504, 511, 484, 501, 511, 510, 543, 528, 490, 494, 492, 499, 507, 526, 544, 555, 504, 491, 509, 541, 489, 502, 523, 538, 536, 537, 532, 532, 522, 506, 523, 522, 503, 492, 476, 523, 479, 531, 515, 520, 492, 516, 526, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[493, 477, 525, 480, 520, 478, 547, 552, 547, 527, 521, 538, 511, 525, 534, 531, 526, 550, 512, 520, 533, 500, 522, 499, 522, 491, 517, 516, 520, 517, 548, 479, 512, 548, 508, 519, 511, 515, 454, 486, 516, 493, 507, 534, 536, 492, 482, 502, 498, 511, 506, 474, 520, 531, 526, 477, 506, 489, 511, 526, 495, 503, 521, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[539, 528, 517, 487, 500, 517, 543, 531, 498, 531, 563, 464, 498, 493, 524, 539, 512, 504, 484, 538, 507, 523, 516, 481, 517, 536, 535, 496, 508, 523, 522, 534, 512, 521, 463, 578, 476, 541, 500, 513, 541, 481, 507, 471, 551, 525, 521, 513, 506, 503, 499, 519, 480, 533, 484, 510, 529, 481, 501, 499, 478, 508, 523, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[553, 506, 483, 572, 513, 474, 492, 492, 519, 493, 535, 503, 492, 520, 541, 508, 549, 521, 513, 506, 562, 516, 519, 517, 442, 500, 507, 504, 509, 560, 504, 519, 511, 542, 520, 517, 542, 554, 502, 524, 491, 486, 479, 522, 525, 519, 524, 486, 512, 510, 493, 486, 553, 507, 470, 471, 544, 505, 540, 481, 524, 502, 478, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[510, 500, 517, 502, 527, 532, 527, 485, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 481, 510, 486, 497, 543, 525, 0, 0, 0, 0, 0, 0, 0, 0, 538, 508, 544, 479, 503, 478, 509, 566, 0, 0, 0, 0, 0, 0, 0, 0, 497, 514, 515, 522, 484, 488, 508, 550, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 493, 515, 533, 544, 490, 491, 0, 0, 0, 0, 0, 0, 0, 0, 498, 526, 519, 532, 515, 511, 502, 473, 8, 1, 0, 0, 0, 0, 0, 0, 503, 515, 484, 504, 503, 565, 515, 487, 32, 3, 0, 0, 0, 0, 0, 0, 494, 484, 533, 541, 505, 502, 499, 510, 12, 10, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4060, 4125, 4078, 4111, 4085, 4111, 4098] avg=4096.0000 max=4125 max_rank=2 min=4060 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 477, 525, 480, 513, 478, 547, 552, 0, 0, 0, 0, 0, 0, 0, 0, 547, 527, 488, 538, 511, 525, 534, 531, 0, 0, 0, 0, 0, 0, 0, 0, 526, 550, 512, 520, 522, 500, 522, 499, 0, 0, 0, 0, 0, 0, 0, 0, 522, 491, 517, 516, 520, 501, 548, 479, 0, 0, 0, 0, 0, 0, 0, 0, 512, 548, 508, 510, 511, 515, 454, 486, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 507, 534, 536, 492, 482, 502, 7, 1, 0, 0, 0, 0, 0, 0, 498, 511, 506, 474, 520, 531, 526, 477, 33, 3, 0, 0, 0, 0, 0, 0, 506, 489, 511, 526, 495, 503, 521, 481, 13, 11, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4201, 4151, 4094, 4044, 4070, 4079, 4064] avg=4096.0000 max=4201 max_rank=1 min=4044 min_rank=4 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16257 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[470, 517, 490, 538, 566, 513, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0, 508, 545, 487, 498, 492, 517, 498, 522, 0, 0, 0, 0, 0, 0, 0, 0, 464, 495, 506, 539, 529, 482, 504, 537, 0, 0, 0, 0, 0, 0, 0, 0, 496, 503, 507, 500, 540, 509, 497, 518, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 493, 515, 525, 479, 494, 499, 0, 0, 0, 0, 0, 0, 0, 0, 517, 514, 530, 490, 510, 511, 488, 537, 8, 1, 0, 0, 0, 0, 0, 0, 487, 519, 516, 470, 515, 529, 502, 541, 33, 3, 0, 0, 0, 0, 0, 0, 526, 533, 473, 534, 488, 525, 574, 514, 13, 11, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4067, 4056, 4070, 4053, 4106, 4115, 4199] avg=4096.0000 max=4199 max_rank=7 min=4053 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[490, 515, 479, 515, 527, 527, 512, 512, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 473, 573, 460, 566, 522, 533, 0, 0, 0, 0, 0, 0, 0, 0, 500, 528, 513, 563, 521, 528, 506, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 507, 519, 503, 522, 515, 540, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 503, 531, 501, 473, 504, 497, 534, 0, 0, 0, 0, 0, 0, 0, 0, 468, 487, 482, 518, 522, 492, 514, 494, 8, 1, 0, 0, 0, 0, 0, 0, 519, 491, 503, 515, 495, 496, 539, 516, 32, 3, 0, 0, 0, 0, 0, 0, 512, 522, 513, 496, 506, 496, 485, 512, 13, 11, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4182, 4163, 4117, 4060, 3986, 4109, 4074] avg=4096.0000 max=4182 max_rank=1 min=3986 min_rank=5 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16229 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[537, 506, 535, 517, 501, 503, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 506, 492, 493, 487, 527, 522, 523, 486, 0, 0, 0, 0, 0, 0, 0, 0, 540, 553, 472, 503, 497, 521, 475, 496, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 519, 542, 519, 495, 515, 557, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 497, 504, 473, 499, 514, 519, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 536, 513, 497, 485, 557, 526, 7, 1, 0, 0, 0, 0, 0, 0, 532, 522, 532, 506, 526, 515, 521, 443, 33, 3, 0, 0, 0, 0, 0, 0, 498, 499, 520, 487, 486, 521, 501, 515, 12, 10, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4036, 4057, 4180, 4046, 4153, 4133, 4057] avg=4096.0000 max=4180 max_rank=3 min=4036 min_rank=1 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[539, 528, 517, 487, 493, 517, 543, 531, 0, 0, 0, 0, 0, 0, 0, 0, 498, 531, 527, 464, 498, 493, 524, 539, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 484, 538, 497, 523, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 535, 496, 508, 507, 522, 534, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 463, 567, 476, 541, 500, 513, 0, 0, 0, 0, 0, 0, 0, 0, 541, 481, 507, 471, 551, 525, 521, 513, 7, 2, 0, 0, 0, 0, 0, 0, 506, 503, 499, 519, 480, 533, 484, 510, 36, 3, 0, 0, 0, 0, 0, 0, 529, 481, 501, 499, 478, 508, 523, 493, 13, 10, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4074, 4055, 4155, 4093, 4119, 4073, 4044] avg=4096.0000 max=4155 max_rank=0 min=4044 min_rank=7 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16329 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[479, 482, 507, 542, 534, 517, 525, 504, 0, 0, 0, 0, 0, 0, 0, 0, 515, 473, 502, 532, 529, 494, 517, 512, 0, 0, 0, 0, 0, 0, 0, 0, 467, 513, 487, 518, 494, 511, 484, 501, 0, 0, 0, 0, 0, 0, 0, 0, 511, 510, 543, 528, 490, 479, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 544, 545, 504, 491, 509, 541, 0, 0, 0, 0, 0, 0, 0, 0, 489, 502, 523, 538, 536, 537, 532, 532, 8, 1, 0, 0, 0, 0, 0, 0, 522, 506, 523, 522, 503, 492, 476, 523, 34, 3, 0, 0, 0, 0, 0, 0, 479, 531, 515, 520, 492, 516, 526, 498, 12, 10, 9, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4074, 3975, 4052, 4167, 4198, 4104, 4108] avg=4096.0000 max=4198 max_rank=5 min=3975 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16577 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[553, 506, 483, 572, 501, 474, 492, 492, 0, 0, 0, 0, 0, 0, 0, 0, 519, 493, 498, 503, 492, 520, 541, 508, 0, 0, 0, 0, 0, 0, 0, 0, 549, 521, 513, 506, 545, 516, 519, 517, 0, 0, 0, 0, 0, 0, 0, 0, 442, 500, 507, 504, 509, 537, 504, 519, 0, 0, 0, 0, 0, 0, 0, 0, 511, 542, 520, 499, 542, 554, 502, 524, 0, 0, 0, 0, 0, 0, 0, 0, 491, 486, 479, 522, 525, 519, 524, 486, 12, 7, 0, 0, 0, 0, 0, 0, 512, 510, 493, 486, 553, 507, 470, 471, 37, 5, 0, 0, 0, 0, 0, 0, 544, 505, 540, 481, 524, 502, 478, 504, 18, 17, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4074, 4186, 4022, 4194, 4051, 4044, 4124] avg=4096.0000 max=4194 max_rank=4 min=4022 min_rank=3 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 3968, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[527, 485, 510, 500, 517, 525, 499, 522, 523, 513, 531, 475, 538, 529, 533, 529, 493, 523, 492, 516, 500, 500, 494, 519, 533, 522, 537, 529, 495, 482, 539, 543, 520, 509, 518, 526, 462, 489, 543, 508, 475, 515, 497, 528, 483, 509, 547, 498, 504, 537, 491, 501, 521, 492, 517, 511, 519, 521, 475, 503, 518, 492, 546, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[512, 541, 461, 518, 467, 503, 482, 519, 532, 490, 514, 533, 526, 535, 506, 555, 509, 534, 502, 492, 503, 487, 503, 497, 496, 493, 512, 517, 540, 521, 516, 513, 512, 512, 527, 539, 515, 504, 545, 518, 500, 506, 534, 532, 469, 493, 492, 502, 515, 524, 541, 524, 495, 515, 512, 532, 488, 491, 538, 514, 494, 514, 515, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[515, 545, 549, 508, 555, 501, 470, 521, 510, 526, 502, 495, 532, 513, 546, 476, 546, 497, 505, 523, 503, 515, 518, 575, 517, 512, 519, 490, 511, 498, 481, 515, 493, 541, 536, 516, 489, 485, 466, 492, 497, 515, 532, 502, 504, 517, 528, 486, 506, 539, 507, 495, 494, 522, 533, 524, 441, 503, 517, 506, 508, 522, 533, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[501, 492, 551, 511, 538, 509, 497, 507, 496, 532, 546, 517, 523, 504, 518, 525, 506, 503, 539, 515, 526, 531, 518, 518, 525, 489, 493, 484, 519, 475, 532, 542, 515, 546, 487, 492, 523, 473, 509, 516, 520, 534, 495, 503, 500, 538, 496, 562, 523, 533, 480, 513, 504, 500, 513, 506, 516, 489, 491, 500, 498, 503, 485, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[525, 532, 528, 516, 557, 543, 568, 522, 544, 526, 513, 463, 534, 524, 547, 483, 508, 523, 508, 527, 479, 505, 528, 513, 525, 509, 520, 526, 478, 518, 538, 502, 467, 536, 500, 509, 518, 515, 509, 514, 496, 493, 513, 510, 518, 530, 472, 469, 533, 479, 566, 484, 486, 506, 507, 489, 478, 504, 481, 509, 510, 524, 511, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[519, 494, 504, 515, 507, 516, 518, 519, 497, 491, 562, 507, 520, 529, 530, 507, 501, 536, 514, 523, 528, 530, 516, 536, 498, 499, 491, 528, 523, 492, 519, 508, 520, 556, 536, 500, 513, 529, 501, 528, 499, 467, 509, 497, 514, 524, 512, 517, 512, 522, 502, 481, 517, 528, 515, 470, 485, 481, 529, 501, 486, 531, 517, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[562, 521, 503, 507, 491, 489, 517, 494, 532, 528, 527, 506, 493, 531, 508, 561, 516, 463, 532, 521, 520, 489, 513, 506, 535, 508, 522, 485, 523, 518, 534, 539, 452, 498, 524, 512, 517, 494, 493, 516, 479, 501, 474, 542, 521, 503, 524, 532, 534, 494, 492, 527, 502, 485, 510, 513, 528, 517, 528, 519, 516, 512, 471, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[498, 510, 497, 504, 518, 503, 501, 504, 490, 510, 530, 527, 530, 529, 537, 503, 549, 567, 509, 508, 475, 516, 476, 487, 489, 501, 569, 489, 509, 505, 493, 500, 522, 542, 464, 495, 523, 532, 533, 522, 519, 527, 488, 517, 515, 495, 496, 528, 514, 543, 503, 497, 529, 515, 506, 517, 533, 520, 480, 531, 526, 493, 494, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 492, 551, 511, 538, 509, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 532, 546, 517, 523, 504, 461, 525, 0, 0, 0, 0, 0, 0, 0, 0, 506, 503, 539, 515, 526, 531, 518, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 489, 492, 484, 519, 475, 532, 542, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 487, 492, 523, 473, 509, 516, 4, 1, 0, 0, 0, 0, 0, 0, 520, 534, 495, 503, 500, 538, 496, 562, 26, 5, 0, 0, 0, 0, 0, 0, 523, 533, 480, 513, 504, 500, 513, 506, 5, 2, 0, 0, 0, 0, 0, 0, 516, 489, 491, 500, 498, 503, 485, 523, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4104, 4150, 4058, 4066, 4179, 4079, 4036] avg=4096.0000 max=4179 max_rank=5 min=4036 min_rank=7 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[517, 485, 510, 500, 517, 525, 499, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 513, 531, 475, 538, 529, 475, 529, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 492, 516, 500, 500, 494, 513, 0, 0, 0, 0, 0, 0, 0, 0, 533, 522, 536, 529, 495, 482, 539, 543, 0, 0, 0, 0, 0, 0, 0, 0, 520, 509, 518, 526, 462, 489, 543, 508, 4, 1, 0, 0, 0, 0, 0, 0, 475, 515, 497, 528, 483, 509, 547, 498, 26, 5, 0, 0, 0, 0, 0, 0, 504, 537, 491, 501, 521, 492, 517, 511, 5, 2, 0, 0, 0, 0, 0, 0, 519, 521, 475, 503, 518, 492, 546, 520, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4113, 4031, 4179, 4080, 4083, 4081, 4126] avg=4096.0000 max=4179 max_rank=3 min=4031 min_rank=2 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[502, 541, 461, 518, 467, 503, 482, 519, 0, 0, 0, 0, 0, 0, 0, 0, 532, 490, 514, 533, 526, 535, 450, 555, 0, 0, 0, 0, 0, 0, 0, 0, 509, 534, 502, 492, 503, 487, 503, 491, 0, 0, 0, 0, 0, 0, 0, 0, 496, 493, 511, 517, 540, 521, 516, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 527, 539, 515, 504, 545, 518, 4, 1, 0, 0, 0, 0, 0, 0, 500, 506, 534, 532, 469, 493, 492, 502, 25, 5, 0, 0, 0, 0, 0, 0, 515, 524, 541, 524, 495, 515, 512, 532, 5, 2, 0, 0, 0, 0, 0, 0, 488, 491, 538, 514, 494, 514, 515, 527, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3993, 4135, 4021, 4107, 4177, 4058, 4165, 4112] avg=4096.0000 max=4177 max_rank=4 min=3993 min_rank=0 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[509, 494, 504, 515, 507, 516, 518, 519, 0, 0, 0, 0, 0, 0, 0, 0, 497, 491, 562, 507, 520, 529, 472, 507, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 514, 523, 528, 530, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 490, 528, 523, 492, 519, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 556, 536, 500, 513, 529, 501, 528, 4, 1, 0, 0, 0, 0, 0, 0, 499, 467, 509, 497, 514, 524, 512, 517, 26, 5, 0, 0, 0, 0, 0, 0, 512, 522, 502, 481, 517, 528, 515, 470, 5, 2, 0, 0, 0, 0, 0, 0, 485, 481, 529, 501, 486, 531, 517, 492, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4085, 4178, 4057, 4188, 4070, 4054, 4054] avg=4096.0000 max=4188 max_rank=4 min=4054 min_rank=6 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[505, 545, 549, 508, 555, 501, 470, 521, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 502, 495, 532, 513, 486, 476, 0, 0, 0, 0, 0, 0, 0, 0, 546, 497, 505, 523, 503, 515, 518, 568, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 518, 490, 511, 498, 481, 515, 0, 0, 0, 0, 0, 0, 0, 0, 493, 541, 536, 516, 489, 485, 466, 492, 5, 1, 0, 0, 0, 0, 0, 0, 497, 515, 532, 502, 504, 517, 528, 486, 27, 5, 0, 0, 0, 0, 0, 0, 506, 539, 507, 495, 494, 522, 533, 524, 5, 2, 0, 0, 0, 0, 0, 0, 441, 503, 517, 506, 508, 522, 533, 530, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4040, 4175, 4042, 4024, 4113, 4127, 4093] avg=4096.0000 max=4175 max_rank=2 min=4024 min_rank=4 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[488, 510, 497, 504, 518, 503, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 530, 527, 530, 529, 477, 503, 0, 0, 0, 0, 0, 0, 0, 0, 549, 567, 509, 508, 475, 516, 476, 481, 0, 0, 0, 0, 0, 0, 0, 0, 489, 501, 567, 489, 509, 505, 493, 500, 0, 0, 0, 0, 0, 0, 0, 0, 522, 542, 464, 495, 523, 532, 533, 522, 4, 2, 0, 0, 0, 0, 0, 0, 519, 527, 488, 517, 515, 495, 496, 528, 27, 5, 0, 0, 0, 0, 0, 0, 514, 543, 503, 497, 529, 515, 506, 517, 5, 2, 0, 0, 0, 0, 0, 0, 533, 520, 480, 531, 526, 493, 494, 516, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4096, 4081, 4053, 4139, 4117, 4131, 4126] avg=4096.0000 max=4139 max_rank=4 min=4025 min_rank=0 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16513 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 532, 528, 516, 557, 543, 568, 522, 0, 0, 0, 0, 0, 0, 0, 0, 544, 526, 513, 463, 534, 524, 487, 483, 0, 0, 0, 0, 0, 0, 0, 0, 508, 523, 508, 527, 479, 505, 528, 507, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 519, 526, 478, 518, 538, 502, 0, 0, 0, 0, 0, 0, 0, 0, 467, 536, 500, 509, 518, 515, 509, 514, 4, 1, 0, 0, 0, 0, 0, 0, 496, 493, 513, 510, 518, 530, 472, 469, 27, 5, 0, 0, 0, 0, 0, 0, 533, 479, 566, 484, 486, 506, 507, 489, 5, 2, 0, 0, 0, 0, 0, 0, 478, 504, 481, 509, 510, 524, 511, 500, 33, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 4074, 4085, 4115, 4073, 4033, 4057, 4050] avg=4096.0000 max=4281 max_rank=0 min=4033 min_rank=5 imbalance_ratio=1.045166 max_load_reduction=28487 max_load_reduction_pct=0.869354 moved_to_echo=16213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 521, 503, 507, 491, 489, 517, 494, 0, 0, 0, 0, 0, 0, 0, 0, 532, 528, 527, 506, 493, 531, 443, 561, 0, 0, 0, 0, 0, 0, 0, 0, 516, 463, 532, 521, 520, 489, 513, 493, 0, 0, 0, 0, 0, 0, 0, 0, 535, 508, 515, 485, 523, 518, 534, 539, 0, 0, 0, 0, 0, 0, 0, 0, 452, 498, 524, 512, 517, 494, 493, 516, 8, 7, 0, 0, 0, 0, 0, 0, 479, 501, 474, 542, 521, 503, 524, 532, 29, 10, 0, 0, 0, 0, 0, 0, 534, 494, 492, 527, 502, 485, 510, 513, 12, 5, 0, 0, 0, 0, 0, 0, 528, 517, 528, 519, 516, 512, 471, 544, 36, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4121, 4047, 4157, 4021, 4115, 4074, 4171] avg=4096.0000 max=4171 max_rank=7 min=4021 min_rank=4 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4352, 4096, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[521, 516, 482, 567, 514, 500, 509, 561, 513, 518, 539, 503, 490, 549, 531, 527, 554, 526, 525, 514, 505, 516, 516, 526, 516, 480, 533, 525, 491, 542, 522, 494, 499, 494, 486, 523, 508, 483, 484, 500, 530, 507, 541, 476, 519, 494, 473, 495, 504, 490, 499, 540, 507, 518, 534, 509, 542, 515, 484, 485, 496, 489, 508, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[478, 529, 525, 541, 525, 532, 514, 540, 498, 503, 507, 515, 507, 565, 498, 527, 508, 500, 503, 539, 510, 535, 521, 528, 506, 514, 538, 537, 511, 484, 542, 556, 500, 516, 517, 515, 492, 496, 520, 508, 520, 501, 497, 513, 514, 505, 493, 521, 502, 505, 527, 498, 491, 478, 469, 497, 521, 483, 500, 546, 476, 489, 488, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[534, 537, 567, 507, 525, 521, 521, 490, 532, 526, 518, 478, 506, 517, 502, 502, 511, 502, 529, 513, 501, 530, 496, 539, 525, 560, 513, 466, 510, 532, 507, 479, 517, 494, 466, 515, 536, 503, 511, 505, 515, 505, 521, 517, 551, 501, 534, 503, 516, 537, 507, 521, 469, 502, 478, 532, 516, 471, 514, 495, 481, 516, 496, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[512, 453, 514, 488, 511, 509, 509, 527, 538, 545, 542, 516, 523, 510, 491, 523, 493, 514, 520, 515, 494, 517, 501, 546, 498, 518, 503, 543, 519, 495, 522, 523, 483, 516, 550, 496, 490, 493, 534, 494, 502, 509, 521, 520, 505, 488, 517, 498, 493, 532, 512, 493, 565, 477, 539, 498, 502, 535, 521, 524, 533, 522, 501, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[521, 517, 546, 501, 538, 473, 501, 529, 477, 525, 485, 544, 493, 550, 506, 523, 495, 486, 508, 536, 547, 494, 521, 528, 531, 509, 559, 491, 497, 556, 545, 519, 516, 491, 465, 503, 511, 526, 531, 512, 511, 535, 477, 487, 509, 483, 522, 503, 483, 511, 503, 508, 515, 539, 494, 500, 512, 527, 523, 494, 519, 502, 496, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[515, 523, 500, 506, 482, 524, 507, 536, 509, 514, 509, 510, 542, 516, 493, 537, 526, 497, 511, 469, 452, 516, 540, 554, 542, 538, 519, 507, 511, 482, 503, 525, 516, 524, 501, 507, 492, 528, 501, 504, 488, 504, 494, 536, 519, 513, 504, 545, 492, 506, 520, 500, 533, 509, 471, 502, 507, 503, 522, 501, 501, 532, 524, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[470, 520, 509, 483, 502, 498, 520, 512, 550, 513, 500, 506, 503, 519, 502, 515, 529, 504, 485, 484, 516, 499, 514, 516, 514, 529, 541, 512, 520, 511, 551, 503, 448, 534, 553, 504, 540, 513, 516, 528, 505, 502, 462, 559, 485, 462, 508, 515, 493, 536, 531, 494, 536, 511, 551, 498, 504, 527, 540, 499, 538, 515, 493, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[498, 534, 510, 535, 528, 522, 517, 525, 480, 528, 518, 518, 522, 535, 515, 506, 529, 494, 487, 521, 504, 531, 526, 520, 527, 493, 485, 509, 499, 511, 530, 485, 513, 517, 500, 500, 526, 525, 506, 542, 520, 507, 502, 526, 533, 500, 518, 493, 511, 519, 514, 478, 489, 533, 508, 505, 500, 515, 540, 490, 507, 504, 472, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 516, 482, 567, 514, 500, 509, 533, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 539, 503, 490, 517, 531, 527, 0, 0, 0, 0, 0, 0, 0, 0, 554, 526, 525, 514, 505, 516, 516, 512, 0, 0, 0, 0, 0, 0, 0, 0, 516, 480, 533, 525, 491, 542, 487, 494, 0, 0, 0, 0, 0, 0, 0, 0, 499, 494, 486, 523, 508, 483, 484, 500, 21, 9, 0, 0, 0, 0, 0, 0, 530, 507, 541, 476, 519, 494, 473, 495, 23, 6, 0, 0, 0, 0, 0, 0, 504, 490, 499, 540, 507, 518, 534, 509, 29, 0, 0, 0, 0, 0, 0, 0, 542, 515, 484, 485, 496, 489, 508, 511, 14, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4138, 4168, 4068, 4007, 4064, 4130, 4051] avg=4096.0000 max=4168 max_rank=2 min=4007 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16252 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[534, 537, 567, 507, 525, 521, 521, 466, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 518, 478, 506, 488, 502, 502, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 529, 513, 501, 530, 496, 525, 0, 0, 0, 0, 0, 0, 0, 0, 525, 560, 513, 466, 510, 532, 473, 479, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 466, 515, 536, 503, 511, 505, 18, 8, 0, 0, 0, 0, 0, 0, 515, 505, 521, 517, 551, 501, 534, 503, 21, 6, 0, 0, 0, 0, 0, 0, 516, 537, 507, 521, 469, 502, 478, 532, 28, 0, 0, 0, 0, 0, 0, 0, 516, 471, 514, 495, 481, 516, 496, 527, 14, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4178, 4052, 4107, 4058, 4073, 4174, 4090, 4036] avg=4096.0000 max=4178 max_rank=0 min=4036 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[521, 517, 546, 501, 538, 473, 501, 504, 0, 0, 0, 0, 0, 0, 0, 0, 477, 525, 485, 544, 493, 518, 506, 523, 0, 0, 0, 0, 0, 0, 0, 0, 495, 486, 508, 536, 547, 494, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 559, 491, 497, 556, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 516, 491, 465, 503, 511, 526, 531, 512, 19, 9, 0, 0, 0, 0, 0, 0, 511, 535, 477, 487, 509, 483, 522, 503, 23, 6, 0, 0, 0, 0, 0, 0, 483, 511, 503, 508, 515, 539, 494, 500, 30, 0, 0, 0, 0, 0, 0, 0, 512, 527, 523, 494, 519, 502, 496, 509, 14, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4071, 4101, 4171, 4083, 4056, 4083, 4102] avg=4096.0000 max=4171 max_rank=3 min=4056 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[478, 529, 525, 541, 525, 532, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 498, 503, 507, 515, 507, 532, 498, 527, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 503, 539, 510, 535, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 538, 537, 511, 484, 506, 556, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 517, 515, 492, 496, 520, 508, 20, 9, 0, 0, 0, 0, 0, 0, 520, 501, 497, 513, 514, 505, 493, 521, 24, 6, 0, 0, 0, 0, 0, 0, 502, 505, 527, 498, 491, 478, 469, 497, 30, 0, 0, 0, 0, 0, 0, 0, 521, 483, 500, 546, 476, 489, 488, 534, 14, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4087, 4130, 4152, 4093, 4094, 3997, 4058] avg=4096.0000 max=4157 max_rank=0 min=3997 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16242 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 453, 514, 488, 511, 509, 509, 502, 0, 0, 0, 0, 0, 0, 0, 0, 538, 545, 542, 516, 523, 481, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 493, 514, 520, 515, 494, 517, 501, 531, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 503, 543, 519, 495, 487, 523, 0, 0, 0, 0, 0, 0, 0, 0, 483, 516, 550, 496, 490, 493, 534, 494, 19, 8, 0, 0, 0, 0, 0, 0, 502, 509, 521, 520, 505, 488, 517, 498, 21, 6, 0, 0, 0, 0, 0, 0, 493, 532, 512, 493, 565, 477, 539, 498, 29, 0, 0, 0, 0, 0, 0, 0, 502, 535, 521, 524, 533, 522, 501, 473, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3998, 4159, 4085, 4086, 4083, 4087, 4138, 4132] avg=4096.0000 max=4159 max_rank=1 min=3998 min_rank=0 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16440 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[498, 534, 510, 535, 528, 522, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 480, 528, 518, 518, 522, 504, 515, 506, 0, 0, 0, 0, 0, 0, 0, 0, 529, 494, 487, 521, 504, 531, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 493, 485, 509, 499, 511, 495, 485, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 500, 500, 526, 525, 506, 542, 19, 9, 0, 0, 0, 0, 0, 0, 520, 507, 502, 526, 533, 500, 518, 493, 22, 6, 0, 0, 0, 0, 0, 0, 511, 519, 514, 478, 489, 533, 508, 505, 29, 0, 0, 0, 0, 0, 0, 0, 500, 515, 540, 490, 507, 504, 472, 513, 14, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4091, 4098, 4004, 4157, 4127, 4086, 4061] avg=4096.0000 max=4157 max_rank=4 min=4004 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[515, 523, 500, 506, 482, 524, 507, 510, 0, 0, 0, 0, 0, 0, 0, 0, 509, 514, 509, 510, 542, 487, 493, 537, 0, 0, 0, 0, 0, 0, 0, 0, 526, 497, 511, 469, 452, 516, 540, 539, 0, 0, 0, 0, 0, 0, 0, 0, 542, 538, 519, 507, 511, 482, 469, 525, 0, 0, 0, 0, 0, 0, 0, 0, 516, 524, 501, 507, 492, 528, 501, 504, 20, 8, 0, 0, 0, 0, 0, 0, 488, 504, 494, 536, 519, 513, 504, 545, 21, 6, 0, 0, 0, 0, 0, 0, 492, 506, 520, 500, 533, 509, 471, 502, 28, 0, 0, 0, 0, 0, 0, 0, 507, 503, 522, 501, 501, 532, 524, 554, 15, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4101, 4050, 4093, 4101, 4130, 4061, 4165] avg=4096.0000 max=4165 max_rank=7 min=4050 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[470, 520, 509, 483, 502, 498, 520, 479, 0, 0, 0, 0, 0, 0, 0, 0, 550, 513, 500, 506, 503, 480, 502, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 504, 485, 484, 516, 499, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 541, 512, 520, 511, 506, 503, 0, 0, 0, 0, 0, 0, 0, 0, 448, 534, 553, 504, 540, 513, 516, 528, 22, 13, 0, 0, 0, 0, 0, 0, 505, 502, 462, 559, 485, 462, 508, 515, 26, 12, 0, 0, 0, 0, 0, 0, 493, 536, 531, 494, 536, 511, 551, 498, 33, 0, 0, 0, 0, 0, 0, 0, 504, 527, 540, 499, 538, 515, 493, 518, 18, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3981, 4069, 4029, 4136, 4171, 4036, 4183, 4163] avg=4096.0000 max=4183 max_rank=6 min=3981 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 3968, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[524, 544, 494, 545, 480, 495, 516, 503, 511, 526, 524, 523, 507, 529, 548, 555, 488, 506, 521, 498, 527, 526, 476, 498, 499, 516, 548, 499, 482, 480, 530, 514, 518, 505, 523, 531, 488, 554, 538, 468, 492, 505, 507, 474, 483, 520, 548, 528, 508, 501, 533, 518, 503, 507, 496, 462, 553, 477, 503, 484, 553, 507, 549, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[495, 513, 517, 542, 500, 532, 542, 540, 502, 540, 499, 474, 508, 517, 525, 504, 511, 487, 509, 527, 524, 516, 527, 492, 505, 535, 500, 513, 520, 486, 492, 502, 512, 505, 521, 565, 483, 513, 521, 500, 498, 478, 520, 524, 523, 506, 531, 527, 512, 532, 552, 497, 516, 483, 546, 468, 519, 460, 521, 512, 528, 488, 514, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[492, 474, 537, 514, 524, 525, 492, 506, 495, 508, 549, 506, 519, 492, 493, 496, 551, 533, 475, 496, 519, 515, 513, 489, 537, 540, 527, 501, 488, 471, 520, 520, 496, 530, 536, 487, 498, 507, 497, 524, 511, 573, 507, 473, 531, 527, 493, 506, 508, 532, 528, 539, 486, 496, 492, 521, 478, 528, 526, 518, 530, 515, 540, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[485, 526, 463, 566, 535, 525, 536, 515, 489, 536, 518, 505, 506, 499, 493, 501, 518, 529, 513, 544, 543, 520, 520, 495, 523, 539, 509, 527, 506, 521, 533, 508, 497, 505, 511, 515, 502, 517, 484, 536, 498, 490, 509, 501, 521, 497, 504, 522, 522, 472, 494, 507, 496, 535, 503, 496, 504, 491, 517, 515, 483, 515, 543, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[526, 463, 485, 502, 551, 506, 539, 532, 533, 493, 498, 494, 501, 517, 544, 534, 519, 511, 510, 510, 535, 521, 534, 524, 498, 553, 503, 485, 498, 501, 496, 462, 513, 505, 528, 544, 520, 519, 541, 504, 513, 488, 520, 471, 509, 503, 531, 478, 526, 517, 515, 467, 501, 538, 520, 528, 528, 482, 475, 533, 514, 514, 527, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[529, 504, 510, 515, 508, 515, 505, 473, 512, 491, 493, 541, 518, 522, 575, 527, 517, 533, 499, 515, 472, 530, 510, 524, 523, 535, 514, 514, 525, 513, 539, 511, 504, 486, 489, 527, 503, 460, 489, 545, 499, 503, 520, 526, 495, 513, 498, 512, 515, 550, 530, 523, 509, 491, 518, 486, 508, 513, 540, 492, 488, 507, 515, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[539, 524, 540, 528, 502, 524, 502, 505, 491, 517, 496, 495, 529, 479, 532, 502, 519, 499, 525, 528, 530, 499, 495, 449, 499, 478, 484, 512, 485, 519, 506, 485, 530, 534, 558, 499, 536, 496, 587, 535, 498, 518, 516, 516, 503, 513, 521, 507, 507, 505, 526, 489, 520, 530, 525, 510, 519, 489, 519, 518, 513, 500, 526, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[500, 518, 549, 567, 488, 492, 506, 522, 483, 516, 498, 518, 527, 502, 528, 554, 516, 537, 508, 535, 528, 468, 530, 535, 536, 530, 508, 501, 525, 539, 502, 486, 504, 501, 511, 536, 492, 455, 526, 487, 535, 510, 485, 498, 527, 539, 511, 508, 464, 502, 520, 497, 510, 491, 530, 533, 516, 510, 497, 510, 502, 480, 512, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[524, 544, 494, 521, 480, 495, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 511, 526, 524, 523, 507, 529, 538, 555, 0, 0, 0, 0, 0, 0, 0, 0, 488, 506, 521, 498, 515, 526, 476, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 548, 499, 482, 480, 530, 514, 10, 3, 0, 0, 0, 0, 0, 0, 518, 505, 523, 521, 488, 554, 538, 468, 0, 0, 0, 0, 0, 0, 0, 0, 492, 505, 507, 474, 483, 520, 548, 528, 19, 0, 0, 0, 0, 0, 0, 0, 508, 501, 533, 518, 503, 507, 496, 462, 9, 5, 0, 0, 0, 0, 0, 0, 553, 477, 503, 484, 553, 507, 549, 500, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4213, 4028, 4081, 4115, 4076, 4042, 4136] avg=4096.0000 max=4213 max_rank=1 min=4028 min_rank=2 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[495, 513, 517, 518, 500, 532, 542, 540, 0, 0, 0, 0, 0, 0, 0, 0, 502, 540, 499, 474, 508, 517, 515, 504, 0, 0, 0, 0, 0, 0, 0, 0, 511, 487, 509, 527, 512, 516, 527, 492, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 500, 513, 520, 486, 492, 502, 10, 3, 0, 0, 0, 0, 0, 0, 512, 505, 521, 554, 483, 513, 521, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 478, 520, 524, 523, 506, 531, 527, 19, 0, 0, 0, 0, 0, 0, 0, 512, 532, 552, 497, 516, 483, 546, 468, 9, 5, 0, 0, 0, 0, 0, 0, 519, 460, 521, 512, 528, 488, 514, 497, 11, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4059, 4081, 4066, 4109, 4126, 4120, 4050] avg=4096.0000 max=4157 max_rank=0 min=4050 min_rank=7 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[485, 526, 463, 541, 535, 525, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 489, 536, 518, 505, 506, 499, 484, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 529, 513, 544, 531, 520, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 523, 539, 509, 527, 506, 521, 533, 508, 9, 3, 0, 0, 0, 0, 0, 0, 497, 505, 511, 505, 502, 517, 484, 536, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 509, 501, 521, 497, 504, 522, 20, 0, 0, 0, 0, 0, 0, 0, 522, 472, 494, 507, 496, 535, 503, 496, 9, 5, 0, 0, 0, 0, 0, 0, 504, 491, 517, 515, 483, 515, 543, 520, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4038, 4170, 4178, 4057, 4062, 4039, 4098] avg=4096.0000 max=4178 max_rank=3 min=4038 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16256 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[492, 474, 537, 492, 524, 525, 492, 506, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 549, 506, 519, 492, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 551, 533, 475, 496, 507, 515, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 537, 540, 527, 501, 488, 471, 520, 520, 9, 3, 0, 0, 0, 0, 0, 0, 496, 530, 536, 478, 498, 507, 497, 524, 0, 0, 0, 0, 0, 0, 0, 0, 511, 573, 507, 473, 531, 527, 493, 506, 18, 0, 0, 0, 0, 0, 0, 0, 508, 532, 528, 539, 486, 496, 492, 521, 9, 4, 0, 0, 0, 0, 0, 0, 478, 528, 526, 518, 530, 515, 540, 518, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4049, 4079, 4116, 4066, 4139, 4115, 4162] avg=4096.0000 max=4162 max_rank=7 min=4042 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16482 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 463, 485, 480, 551, 506, 539, 532, 0, 0, 0, 0, 0, 0, 0, 0, 533, 493, 498, 494, 501, 517, 534, 534, 0, 0, 0, 0, 0, 0, 0, 0, 519, 511, 510, 510, 523, 521, 534, 524, 0, 0, 0, 0, 0, 0, 0, 0, 498, 553, 503, 485, 498, 501, 496, 462, 10, 3, 0, 0, 0, 0, 0, 0, 513, 505, 528, 534, 520, 519, 541, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 520, 471, 509, 503, 531, 478, 18, 0, 0, 0, 0, 0, 0, 0, 526, 517, 515, 467, 501, 538, 520, 528, 9, 4, 0, 0, 0, 0, 0, 0, 528, 482, 475, 533, 514, 514, 527, 518, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4104, 4152, 4009, 4164, 4031, 4125, 4101] avg=4096.0000 max=4164 max_rank=4 min=4009 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[500, 518, 549, 542, 488, 492, 506, 522, 0, 0, 0, 0, 0, 0, 0, 0, 483, 516, 498, 518, 527, 502, 518, 554, 0, 0, 0, 0, 0, 0, 0, 0, 516, 537, 508, 535, 516, 468, 530, 535, 0, 0, 0, 0, 0, 0, 0, 0, 536, 530, 508, 501, 525, 539, 502, 486, 10, 3, 0, 0, 0, 0, 0, 0, 504, 501, 511, 526, 492, 455, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 535, 510, 485, 498, 527, 539, 511, 508, 20, 0, 0, 0, 0, 0, 0, 0, 464, 502, 520, 497, 510, 491, 530, 533, 9, 5, 0, 0, 0, 0, 0, 0, 516, 510, 497, 510, 502, 480, 512, 517, 10, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4116, 4145, 4140, 4002, 4133, 4061, 4054] avg=4096.0000 max=4145 max_rank=2 min=4002 min_rank=4 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16250 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 504, 510, 483, 508, 515, 505, 473, 0, 0, 0, 0, 0, 0, 0, 0, 512, 491, 493, 541, 518, 522, 554, 527, 0, 0, 0, 0, 0, 0, 0, 0, 517, 533, 499, 515, 453, 530, 510, 524, 0, 0, 0, 0, 0, 0, 0, 0, 523, 535, 514, 514, 525, 513, 539, 511, 14, 9, 0, 0, 0, 0, 0, 0, 504, 486, 489, 513, 503, 460, 489, 545, 0, 0, 0, 0, 0, 0, 0, 0, 499, 503, 520, 526, 495, 513, 498, 512, 24, 0, 0, 0, 0, 0, 0, 0, 515, 550, 530, 523, 509, 491, 518, 486, 10, 8, 0, 0, 0, 0, 0, 0, 508, 513, 540, 492, 488, 507, 515, 502, 14, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4027, 4158, 4081, 4197, 3989, 4090, 4140, 4086] avg=4096.0000 max=4197 max_rank=3 min=3989 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[539, 524, 540, 504, 502, 524, 502, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 517, 496, 495, 529, 479, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 525, 528, 518, 499, 495, 449, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 484, 512, 485, 519, 506, 485, 10, 3, 0, 0, 0, 0, 0, 0, 530, 534, 558, 490, 536, 496, 587, 535, 0, 0, 0, 0, 0, 0, 0, 0, 498, 518, 516, 516, 503, 513, 521, 507, 19, 0, 0, 0, 0, 0, 0, 0, 507, 505, 526, 489, 520, 530, 525, 510, 9, 5, 0, 0, 0, 0, 0, 0, 519, 489, 519, 518, 513, 500, 526, 488, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4031, 4032, 3981, 4266, 4111, 4126, 4081] avg=4096.0000 max=4266 max_rank=4 min=3981 min_rank=3 imbalance_ratio=1.041504 max_load_reduction=28502 max_load_reduction_pct=0.869812 moved_to_echo=16584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4096, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4352, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 14:57:42] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 277.6 | throughput per GPU (TFLOP/s/GPU): 444.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.030488E+01 | loss scale: 1.0 | grad norm: 8.103 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[541, 543, 471, 502, 509, 500, 529, 503, 481, 509, 490, 495, 525, 564, 522, 504, 509, 487, 549, 521, 557, 476, 464, 509, 526, 523, 524, 516, 509, 477, 490, 484, 464, 513, 513, 483, 515, 508, 506, 545, 497, 528, 476, 532, 525, 547, 522, 520, 489, 483, 529, 532, 542, 488, 555, 519, 513, 533, 490, 498, 531, 523, 525, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[516, 518, 512, 539, 522, 507, 507, 497, 520, 541, 517, 507, 509, 486, 519, 528, 514, 541, 491, 505, 496, 549, 506, 536, 493, 512, 507, 495, 490, 489, 543, 482, 544, 560, 525, 508, 513, 514, 515, 513, 501, 512, 518, 499, 511, 502, 484, 520, 494, 519, 501, 547, 495, 531, 512, 514, 531, 468, 508, 490, 497, 530, 481, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[506, 493, 552, 511, 492, 522, 521, 508, 520, 506, 499, 531, 505, 508, 508, 495, 495, 540, 507, 542, 494, 502, 502, 514, 541, 477, 506, 529, 479, 531, 482, 512, 502, 528, 530, 512, 527, 497, 507, 515, 510, 480, 542, 497, 514, 513, 505, 523, 529, 517, 497, 481, 575, 505, 525, 511, 475, 529, 559, 478, 536, 481, 528, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[493, 527, 517, 520, 545, 498, 536, 536, 504, 546, 540, 515, 516, 513, 510, 499, 515, 538, 522, 483, 501, 495, 525, 532, 485, 507, 537, 521, 476, 565, 517, 491, 470, 542, 514, 495, 528, 503, 524, 503, 526, 506, 491, 519, 494, 484, 494, 486, 538, 516, 525, 496, 482, 549, 527, 521, 513, 491, 497, 501, 515, 490, 481, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[503, 498, 505, 506, 502, 532, 501, 529, 507, 485, 502, 532, 496, 501, 498, 541, 484, 511, 562, 529, 492, 481, 491, 554, 532, 507, 510, 522, 510, 563, 506, 478, 518, 508, 478, 545, 484, 471, 466, 549, 533, 490, 518, 544, 478, 502, 520, 529, 566, 525, 479, 489, 543, 528, 524, 476, 513, 555, 519, 484, 515, 504, 527, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[513, 529, 471, 516, 490, 498, 515, 451, 523, 505, 509, 508, 541, 533, 550, 533, 536, 519, 526, 540, 519, 535, 484, 487, 506, 534, 517, 509, 494, 540, 524, 538, 529, 488, 525, 497, 515, 481, 487, 519, 473, 505, 485, 491, 520, 517, 535, 509, 527, 510, 530, 505, 490, 491, 518, 484, 533, 507, 503, 563, 505, 497, 504, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[525, 531, 489, 536, 503, 491, 517, 507, 490, 516, 528, 492, 535, 524, 531, 505, 558, 547, 500, 547, 551, 498, 511, 525, 503, 488, 495, 480, 511, 522, 510, 511, 527, 539, 524, 490, 518, 528, 540, 537, 507, 494, 508, 496, 528, 530, 510, 520, 509, 482, 490, 494, 521, 537, 485, 495, 485, 483, 520, 478, 502, 496, 507, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[573, 545, 499, 502, 551, 494, 534, 530, 555, 491, 510, 512, 510, 489, 507, 531, 544, 476, 517, 550, 521, 574, 501, 499, 545, 493, 495, 476, 509, 504, 539, 547, 479, 532, 508, 499, 498, 487, 482, 517, 502, 513, 506, 498, 476, 539, 490, 491, 541, 520, 483, 525, 491, 516, 478, 493, 524, 542, 521, 559, 501, 456, 497, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[516, 505, 512, 539, 522, 507, 507, 497, 0, 0, 0, 0, 0, 0, 0, 0, 520, 541, 517, 507, 509, 486, 501, 528, 0, 0, 0, 0, 0, 0, 0, 0, 514, 541, 491, 468, 496, 549, 506, 536, 0, 0, 0, 0, 0, 0, 0, 0, 493, 512, 507, 495, 490, 489, 543, 482, 8, 8, 0, 0, 0, 0, 0, 0, 544, 560, 525, 508, 513, 514, 515, 513, 5, 2, 0, 0, 0, 0, 0, 0, 501, 512, 518, 499, 511, 502, 484, 520, 27, 0, 0, 0, 0, 0, 0, 0, 492, 519, 501, 547, 495, 531, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 531, 468, 508, 490, 497, 530, 481, 517, 10, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4109, 4101, 4027, 4199, 4074, 4111, 4042] avg=4096.0000 max=4199 max_rank=4 min=4027 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[506, 481, 552, 511, 492, 522, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 520, 506, 499, 531, 505, 508, 490, 495, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 507, 503, 494, 502, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 541, 477, 506, 529, 479, 531, 482, 512, 8, 7, 0, 0, 0, 0, 0, 0, 502, 528, 530, 512, 527, 497, 507, 515, 5, 2, 0, 0, 0, 0, 0, 0, 510, 480, 542, 497, 514, 513, 505, 523, 29, 0, 0, 0, 0, 0, 0, 0, 527, 517, 497, 481, 575, 505, 525, 511, 0, 0, 0, 0, 0, 0, 0, 0, 475, 529, 559, 478, 536, 481, 528, 510, 10, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4054, 4057, 4072, 4125, 4113, 4138, 4116] avg=4096.0000 max=4138 max_rank=6 min=4054 min_rank=1 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[541, 530, 471, 502, 509, 500, 529, 503, 0, 0, 0, 0, 0, 0, 0, 0, 481, 509, 490, 495, 525, 564, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 549, 483, 557, 476, 464, 509, 0, 0, 0, 0, 0, 0, 0, 0, 526, 523, 524, 516, 509, 477, 490, 484, 8, 8, 0, 0, 0, 0, 0, 0, 464, 513, 513, 483, 515, 508, 506, 545, 5, 2, 0, 0, 0, 0, 0, 0, 497, 528, 476, 532, 525, 547, 522, 520, 28, 0, 0, 0, 0, 0, 0, 0, 487, 483, 529, 532, 542, 488, 555, 519, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 490, 498, 531, 523, 525, 515, 10, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4072, 4034, 4065, 4054, 4175, 4135, 4148] avg=4096.0000 max=4175 max_rank=5 min=4034 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16512 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[493, 514, 517, 520, 545, 498, 536, 536, 0, 0, 0, 0, 0, 0, 0, 0, 504, 546, 540, 515, 516, 513, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 515, 538, 522, 448, 501, 495, 525, 532, 0, 0, 0, 0, 0, 0, 0, 0, 485, 507, 537, 521, 476, 565, 517, 491, 8, 8, 0, 0, 0, 0, 0, 0, 470, 542, 514, 495, 528, 503, 524, 503, 5, 2, 0, 0, 0, 0, 0, 0, 526, 506, 491, 519, 494, 484, 494, 486, 26, 0, 0, 0, 0, 0, 0, 0, 536, 516, 525, 496, 482, 549, 527, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 497, 501, 515, 490, 481, 522, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4125, 4076, 4115, 4086, 4026, 4152, 4029] avg=4096.0000 max=4159 max_rank=0 min=4026 min_rank=5 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16293 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 486, 505, 506, 502, 532, 501, 529, 0, 0, 0, 0, 0, 0, 0, 0, 507, 485, 502, 532, 496, 501, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 484, 511, 562, 490, 492, 481, 491, 554, 0, 0, 0, 0, 0, 0, 0, 0, 532, 507, 510, 522, 510, 563, 506, 478, 8, 7, 0, 0, 0, 0, 0, 0, 518, 508, 478, 545, 484, 471, 466, 549, 5, 2, 0, 0, 0, 0, 0, 0, 533, 490, 518, 544, 478, 502, 520, 529, 29, 0, 0, 0, 0, 0, 0, 0, 564, 525, 479, 489, 543, 528, 524, 476, 0, 0, 0, 0, 0, 0, 0, 0, 513, 555, 519, 484, 515, 504, 527, 518, 10, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4044, 4065, 4143, 4026, 4143, 4128, 4155] avg=4096.0000 max=4155 max_rank=7 min=4026 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16452 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[525, 518, 489, 536, 503, 491, 517, 507, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 528, 492, 535, 524, 513, 505, 0, 0, 0, 0, 0, 0, 0, 0, 558, 547, 500, 506, 551, 498, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 503, 488, 495, 480, 511, 522, 510, 511, 8, 8, 0, 0, 0, 0, 0, 0, 527, 539, 524, 490, 518, 528, 540, 537, 5, 2, 0, 0, 0, 0, 0, 0, 507, 494, 508, 496, 528, 530, 510, 520, 30, 0, 0, 0, 0, 0, 0, 0, 507, 482, 490, 494, 521, 537, 485, 495, 0, 0, 0, 0, 0, 0, 0, 0, 485, 483, 520, 478, 502, 496, 507, 511, 10, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4103, 4196, 4036, 4210, 4123, 4011, 4003] avg=4096.0000 max=4210 max_rank=4 min=4003 min_rank=7 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[513, 516, 471, 516, 490, 498, 515, 451, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 509, 508, 541, 533, 530, 533, 0, 0, 0, 0, 0, 0, 0, 0, 536, 519, 526, 501, 519, 535, 484, 487, 0, 0, 0, 0, 0, 0, 0, 0, 506, 534, 517, 509, 494, 540, 524, 538, 9, 8, 0, 0, 0, 0, 0, 0, 529, 488, 525, 497, 515, 481, 487, 519, 5, 2, 0, 0, 0, 0, 0, 0, 473, 505, 485, 491, 520, 517, 535, 509, 29, 0, 0, 0, 0, 0, 0, 0, 525, 510, 530, 505, 490, 491, 518, 484, 0, 0, 0, 0, 0, 0, 0, 0, 533, 507, 503, 563, 505, 497, 504, 532, 11, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3970, 4182, 4107, 4179, 4048, 4064, 4053, 4165] avg=4096.0000 max=4182 max_rank=1 min=3970 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16330 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[573, 523, 499, 502, 551, 494, 534, 530, 0, 0, 0, 0, 0, 0, 0, 0, 555, 491, 510, 512, 510, 489, 481, 531, 0, 0, 0, 0, 0, 0, 0, 0, 544, 476, 517, 500, 521, 574, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 545, 493, 495, 476, 509, 504, 539, 547, 12, 11, 0, 0, 0, 0, 0, 0, 479, 532, 508, 499, 498, 487, 482, 517, 11, 7, 0, 0, 0, 0, 0, 0, 502, 513, 506, 498, 476, 539, 490, 491, 35, 0, 0, 0, 0, 0, 0, 0, 534, 520, 483, 525, 491, 516, 478, 493, 0, 0, 0, 0, 0, 0, 0, 0, 524, 542, 521, 559, 501, 456, 497, 481, 14, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4079, 4132, 4131, 4020, 4050, 4040, 4110] avg=4096.0000 max=4206 max_rank=0 min=4020 min_rank=4 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[527, 496, 535, 536, 520, 496, 507, 520, 485, 550, 511, 505, 552, 485, 467, 492, 540, 534, 496, 490, 554, 525, 509, 536, 495, 516, 489, 543, 526, 549, 478, 480, 481, 531, 504, 533, 493, 540, 518, 511, 522, 517, 515, 502, 541, 524, 507, 513, 500, 499, 494, 486, 504, 482, 525, 537, 522, 523, 463, 516, 521, 492, 518, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[493, 522, 537, 503, 518, 522, 480, 523, 556, 514, 503, 512, 505, 522, 538, 554, 521, 502, 555, 529, 504, 543, 505, 566, 509, 522, 537, 500, 543, 520, 495, 523, 463, 507, 511, 481, 500, 514, 481, 516, 509, 504, 495, 546, 501, 480, 495, 511, 506, 529, 484, 536, 499, 548, 485, 503, 535, 516, 475, 508, 488, 481, 497, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[495, 531, 511, 513, 524, 501, 526, 497, 516, 472, 546, 542, 508, 499, 525, 522, 525, 496, 514, 512, 504, 529, 486, 547, 541, 494, 489, 552, 506, 496, 519, 534, 551, 517, 519, 482, 520, 518, 518, 470, 495, 494, 482, 490, 526, 513, 516, 509, 511, 465, 545, 499, 491, 572, 523, 502, 477, 499, 541, 523, 526, 455, 549, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[524, 526, 549, 491, 481, 513, 524, 480, 545, 494, 541, 525, 496, 527, 535, 541, 506, 470, 484, 529, 585, 510, 476, 520, 510, 478, 493, 503, 502, 543, 540, 519, 523, 522, 533, 462, 516, 492, 504, 528, 516, 497, 538, 530, 528, 523, 540, 519, 473, 500, 507, 482, 518, 513, 517, 503, 521, 488, 477, 501, 502, 521, 507, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[488, 551, 493, 529, 481, 509, 502, 531, 494, 510, 494, 507, 526, 505, 506, 525, 498, 536, 518, 495, 524, 513, 512, 553, 495, 538, 508, 495, 548, 547, 508, 489, 518, 487, 514, 480, 491, 492, 508, 497, 471, 519, 531, 469, 499, 487, 512, 501, 526, 527, 510, 561, 532, 548, 493, 520, 534, 527, 548, 477, 487, 516, 536, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[480, 488, 489, 536, 503, 539, 545, 504, 535, 558, 517, 518, 545, 518, 526, 495, 456, 491, 487, 556, 487, 491, 522, 514, 509, 572, 545, 504, 501, 507, 493, 494, 538, 480, 483, 509, 501, 522, 510, 516, 506, 479, 537, 520, 492, 497, 513, 518, 501, 521, 501, 531, 546, 531, 529, 546, 502, 479, 505, 522, 494, 508, 503, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[517, 527, 530, 532, 516, 508, 516, 550, 489, 485, 515, 490, 562, 499, 532, 481, 507, 499, 493, 536, 526, 529, 524, 537, 525, 509, 531, 531, 524, 513, 483, 497, 532, 513, 519, 484, 510, 523, 479, 458, 495, 512, 533, 478, 526, 520, 517, 504, 497, 516, 500, 515, 555, 479, 490, 516, 513, 517, 528, 505, 490, 502, 517, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[499, 536, 512, 485, 518, 482, 518, 529, 511, 466, 544, 535, 542, 511, 531, 490, 482, 507, 538, 501, 508, 517, 500, 511, 508, 540, 527, 512, 494, 495, 540, 519, 531, 472, 469, 493, 526, 540, 525, 526, 512, 541, 479, 503, 538, 534, 505, 494, 519, 504, 514, 496, 543, 521, 551, 507, 480, 464, 485, 518, 537, 511, 506, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[527, 485, 535, 536, 520, 496, 507, 520, 0, 0, 0, 0, 0, 0, 0, 0, 485, 550, 511, 505, 516, 485, 467, 492, 0, 0, 0, 0, 0, 0, 0, 0, 540, 534, 496, 490, 554, 525, 509, 512, 0, 0, 0, 0, 0, 0, 0, 0, 495, 516, 489, 543, 526, 526, 478, 480, 0, 0, 0, 0, 0, 0, 0, 0, 481, 531, 504, 533, 493, 540, 518, 511, 23, 12, 9, 0, 0, 0, 0, 0, 522, 517, 515, 502, 541, 524, 507, 513, 11, 3, 0, 0, 0, 0, 0, 0, 500, 499, 494, 486, 504, 470, 525, 537, 0, 0, 0, 0, 0, 0, 0, 0, 522, 523, 463, 516, 521, 492, 518, 490, 36, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4011, 4160, 4053, 4155, 4155, 4015, 4093] avg=4096.0000 max=4160 max_rank=2 min=4011 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[524, 514, 549, 491, 481, 513, 524, 480, 0, 0, 0, 0, 0, 0, 0, 0, 545, 494, 541, 525, 464, 527, 535, 541, 0, 0, 0, 0, 0, 0, 0, 0, 506, 470, 484, 529, 585, 510, 476, 496, 0, 0, 0, 0, 0, 0, 0, 0, 510, 478, 493, 503, 502, 520, 540, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 533, 462, 516, 492, 504, 528, 23, 12, 10, 0, 0, 0, 0, 0, 516, 497, 538, 530, 528, 523, 540, 519, 12, 3, 0, 0, 0, 0, 0, 0, 473, 500, 507, 482, 518, 500, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 521, 488, 477, 501, 502, 521, 507, 507, 32, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4172, 4056, 4065, 4125, 4206, 4000, 4068] avg=4096.0000 max=4206 max_rank=5 min=4000 min_rank=6 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[493, 511, 537, 503, 518, 522, 480, 523, 0, 0, 0, 0, 0, 0, 0, 0, 556, 514, 503, 512, 472, 522, 538, 554, 0, 0, 0, 0, 0, 0, 0, 0, 521, 502, 555, 529, 504, 543, 505, 540, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 537, 500, 543, 498, 495, 523, 0, 0, 0, 0, 0, 0, 0, 0, 463, 507, 511, 481, 500, 514, 481, 516, 22, 13, 10, 0, 0, 0, 0, 0, 509, 504, 495, 546, 501, 480, 495, 511, 11, 4, 0, 0, 0, 0, 0, 0, 506, 529, 484, 536, 499, 534, 485, 503, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 475, 508, 488, 481, 497, 488, 33, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4171, 4199, 4127, 4018, 4056, 4076, 4034] avg=4096.0000 max=4199 max_rank=2 min=4018 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16184 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[488, 539, 493, 529, 481, 509, 502, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 510, 494, 507, 492, 505, 506, 525, 0, 0, 0, 0, 0, 0, 0, 0, 498, 536, 518, 495, 524, 513, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 538, 508, 495, 548, 524, 508, 489, 0, 0, 0, 0, 0, 0, 0, 0, 518, 487, 514, 480, 491, 492, 508, 497, 23, 13, 10, 0, 0, 0, 0, 0, 471, 519, 531, 469, 499, 487, 512, 501, 12, 4, 0, 0, 0, 0, 0, 0, 526, 527, 510, 561, 532, 534, 493, 520, 0, 0, 0, 0, 0, 0, 0, 0, 534, 527, 548, 477, 487, 516, 536, 522, 34, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4033, 4124, 4105, 4033, 4005, 4203, 4193] avg=4096.0000 max=4203 max_rank=6 min=4005 min_rank=5 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[480, 477, 489, 536, 503, 539, 545, 504, 0, 0, 0, 0, 0, 0, 0, 0, 535, 558, 517, 518, 510, 518, 526, 495, 0, 0, 0, 0, 0, 0, 0, 0, 456, 491, 487, 556, 487, 491, 522, 491, 0, 0, 0, 0, 0, 0, 0, 0, 509, 572, 545, 504, 501, 486, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 538, 480, 483, 509, 501, 522, 510, 516, 21, 12, 10, 0, 0, 0, 0, 0, 506, 479, 537, 520, 492, 497, 513, 518, 11, 4, 0, 0, 0, 0, 0, 0, 501, 521, 501, 531, 546, 517, 529, 546, 0, 0, 0, 0, 0, 0, 0, 0, 502, 479, 505, 522, 494, 508, 503, 503, 35, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4177, 3981, 4104, 4102, 4077, 4192, 4062] avg=4096.0000 max=4192 max_rank=6 min=3981 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 519, 511, 513, 524, 501, 526, 497, 0, 0, 0, 0, 0, 0, 0, 0, 516, 472, 546, 542, 475, 499, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 525, 496, 514, 512, 504, 529, 486, 522, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 489, 552, 506, 475, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 551, 517, 519, 482, 520, 518, 518, 470, 21, 13, 11, 0, 0, 0, 0, 0, 495, 494, 482, 490, 526, 513, 516, 509, 12, 4, 0, 0, 0, 0, 0, 0, 511, 465, 545, 499, 491, 557, 523, 502, 0, 0, 0, 0, 0, 0, 0, 0, 477, 499, 541, 523, 526, 455, 549, 498, 33, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4097, 4088, 4110, 4140, 4041, 4093, 4113] avg=4096.0000 max=4140 max_rank=4 min=4041 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 512, 530, 532, 516, 508, 516, 550, 0, 0, 0, 0, 0, 0, 0, 0, 489, 485, 515, 490, 521, 499, 532, 481, 0, 0, 0, 0, 0, 0, 0, 0, 507, 499, 493, 536, 526, 529, 524, 505, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 531, 531, 524, 490, 483, 497, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 519, 484, 510, 523, 479, 458, 23, 15, 14, 0, 0, 0, 0, 0, 495, 512, 533, 478, 526, 520, 517, 504, 15, 7, 0, 0, 0, 0, 0, 0, 497, 516, 500, 515, 555, 458, 490, 516, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 528, 505, 490, 502, 517, 512, 41, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4181, 4012, 4119, 4090, 4070, 4107, 4047, 4142] avg=4096.0000 max=4181 max_rank=0 min=4012 min_rank=1 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 524, 512, 485, 518, 482, 518, 529, 0, 0, 0, 0, 0, 0, 0, 0, 511, 466, 544, 535, 507, 511, 531, 490, 0, 0, 0, 0, 0, 0, 0, 0, 482, 507, 538, 501, 508, 517, 500, 488, 0, 0, 0, 0, 0, 0, 0, 0, 508, 540, 527, 512, 494, 474, 540, 519, 0, 0, 0, 0, 0, 0, 0, 0, 531, 472, 469, 493, 526, 540, 525, 526, 21, 12, 10, 0, 0, 0, 0, 0, 512, 541, 479, 503, 538, 534, 505, 494, 12, 3, 0, 0, 0, 0, 0, 0, 519, 504, 514, 496, 543, 508, 551, 507, 0, 0, 0, 0, 0, 0, 0, 0, 480, 464, 485, 518, 537, 511, 506, 516, 35, 11, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4095, 4041, 4114, 4125, 4121, 4142, 4063] avg=4096.0000 max=4142 max_rank=6 min=4041 min_rank=2 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[536, 539, 489, 510, 539, 503, 508, 543, 489, 474, 511, 475, 512, 512, 550, 528, 495, 481, 492, 497, 495, 458, 562, 482, 557, 531, 585, 541, 524, 528, 518, 520, 495, 502, 521, 491, 546, 505, 478, 511, 506, 540, 521, 482, 539, 494, 497, 483, 547, 547, 492, 502, 520, 527, 494, 484, 520, 550, 503, 507, 516, 476, 480, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[534, 511, 518, 492, 514, 531, 509, 537, 527, 532, 524, 509, 513, 515, 520, 493, 552, 499, 485, 492, 486, 525, 514, 541, 492, 504, 491, 502, 501, 505, 482, 517, 503, 538, 515, 524, 486, 491, 492, 513, 482, 514, 517, 506, 515, 515, 516, 503, 527, 526, 493, 523, 528, 484, 503, 526, 495, 518, 507, 516, 538, 509, 533, 545] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[525, 522, 509, 559, 506, 494, 510, 477, 504, 521, 471, 495, 488, 516, 479, 520, 511, 535, 508, 526, 508, 500, 510, 512, 477, 475, 529, 502, 503, 514, 513, 468, 494, 504, 498, 510, 523, 498, 512, 524, 522, 535, 510, 536, 517, 535, 498, 539, 522, 522, 500, 555, 528, 500, 474, 595, 521, 492, 515, 539, 510, 493, 522, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[471, 528, 552, 519, 496, 515, 537, 533, 534, 511, 525, 523, 518, 542, 524, 534, 495, 518, 539, 498, 476, 496, 522, 473, 544, 499, 522, 495, 507, 478, 518, 544, 512, 492, 516, 506, 506, 572, 518, 494, 485, 516, 516, 518, 531, 500, 500, 554, 511, 503, 519, 490, 553, 476, 494, 521, 486, 519, 459, 527, 476, 501, 510, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[529, 528, 501, 505, 532, 503, 527, 501, 520, 489, 550, 566, 566, 518, 481, 517, 509, 509, 520, 514, 515, 538, 517, 522, 504, 498, 485, 539, 469, 508, 495, 516, 541, 522, 484, 552, 489, 523, 479, 514, 498, 525, 529, 525, 538, 505, 510, 475, 476, 479, 549, 502, 506, 515, 507, 498, 507, 514, 505, 492, 507, 496, 507, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[521, 494, 514, 532, 495, 514, 460, 507, 516, 514, 499, 525, 539, 469, 543, 514, 509, 515, 503, 490, 515, 539, 557, 524, 543, 513, 542, 499, 457, 512, 505, 517, 489, 505, 461, 491, 493, 527, 509, 511, 521, 471, 521, 506, 516, 525, 512, 528, 487, 541, 505, 507, 508, 543, 551, 474, 502, 515, 521, 525, 494, 536, 504, 573] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[525, 533, 527, 521, 555, 532, 538, 479, 532, 502, 502, 520, 506, 512, 524, 491, 549, 536, 511, 510, 513, 474, 505, 527, 485, 532, 511, 520, 525, 508, 519, 516, 530, 512, 500, 521, 496, 498, 496, 527, 511, 496, 557, 501, 494, 535, 479, 512, 558, 496, 517, 512, 494, 476, 504, 524, 499, 474, 494, 465, 513, 510, 544, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[462, 527, 523, 488, 506, 485, 512, 525, 524, 479, 510, 503, 490, 501, 539, 492, 491, 515, 500, 490, 502, 510, 482, 550, 556, 512, 499, 509, 537, 532, 546, 514, 540, 536, 491, 530, 486, 524, 508, 483, 511, 536, 499, 518, 545, 499, 533, 494, 509, 457, 501, 509, 495, 513, 538, 526, 512, 534, 536, 541, 485, 541, 492, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 471, 514, 532, 495, 514, 460, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 514, 499, 525, 539, 469, 535, 514, 0, 0, 0, 0, 0, 0, 0, 0, 509, 515, 503, 490, 515, 539, 557, 524, 10, 4, 0, 0, 0, 0, 0, 0, 543, 513, 538, 499, 457, 512, 505, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 461, 491, 493, 527, 509, 511, 23, 1, 0, 0, 0, 0, 0, 0, 521, 471, 521, 506, 505, 525, 512, 528, 0, 0, 0, 0, 0, 0, 0, 0, 487, 541, 505, 507, 508, 543, 551, 474, 0, 0, 0, 0, 0, 0, 0, 0, 502, 515, 521, 525, 494, 536, 504, 573, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4014, 4111, 4166, 4084, 4010, 4089, 4116, 4178] avg=4096.0000 max=4178 max_rank=7 min=4010 min_rank=4 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[536, 514, 489, 510, 539, 503, 508, 543, 0, 0, 0, 0, 0, 0, 0, 0, 489, 474, 511, 475, 512, 512, 541, 528, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 492, 497, 495, 458, 562, 482, 11, 4, 0, 0, 0, 0, 0, 0, 557, 531, 580, 541, 524, 528, 518, 520, 0, 0, 0, 0, 0, 0, 0, 0, 495, 502, 521, 491, 546, 505, 478, 511, 25, 1, 0, 0, 0, 0, 0, 0, 506, 540, 521, 482, 527, 494, 497, 483, 0, 0, 0, 0, 0, 0, 0, 0, 547, 547, 492, 502, 520, 527, 494, 484, 0, 0, 0, 0, 0, 0, 0, 0, 520, 550, 503, 507, 516, 476, 480, 508, 5, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4042, 3977, 4299, 4075, 4050, 4113, 4070] avg=4096.0000 max=4299 max_rank=3 min=3977 min_rank=2 imbalance_ratio=1.049561 max_load_reduction=28469 max_load_reduction_pct=0.868805 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[534, 487, 518, 492, 514, 531, 509, 537, 0, 0, 0, 0, 0, 0, 0, 0, 527, 532, 524, 509, 513, 515, 512, 493, 0, 0, 0, 0, 0, 0, 0, 0, 552, 499, 485, 492, 486, 525, 514, 541, 10, 4, 0, 0, 0, 0, 0, 0, 492, 504, 487, 502, 501, 505, 482, 517, 0, 0, 0, 0, 0, 0, 0, 0, 503, 538, 515, 524, 486, 491, 492, 513, 24, 1, 0, 0, 0, 0, 0, 0, 482, 514, 517, 506, 504, 515, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 527, 526, 493, 523, 528, 484, 503, 526, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 507, 516, 538, 509, 533, 545, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4125, 4108, 3990, 4087, 4057, 4110, 4169] avg=4096.0000 max=4169 max_rank=7 min=3990 min_rank=3 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 503, 501, 505, 532, 503, 527, 501, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 550, 566, 566, 518, 473, 517, 0, 0, 0, 0, 0, 0, 0, 0, 509, 509, 520, 514, 515, 538, 517, 522, 11, 4, 0, 0, 0, 0, 0, 0, 504, 498, 481, 539, 469, 508, 495, 516, 0, 0, 0, 0, 0, 0, 0, 0, 541, 522, 484, 552, 489, 523, 479, 514, 25, 1, 0, 0, 0, 0, 0, 0, 498, 525, 529, 525, 526, 505, 510, 475, 0, 0, 0, 0, 0, 0, 0, 0, 476, 479, 549, 502, 506, 515, 507, 498, 0, 0, 0, 0, 0, 0, 0, 0, 507, 514, 505, 492, 507, 496, 507, 508, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4199, 4159, 4010, 4130, 4093, 4032, 4044] avg=4096.0000 max=4199 max_rank=1 min=4010 min_rank=3 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16299 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[525, 498, 509, 559, 506, 494, 510, 477, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 471, 495, 488, 516, 471, 520, 0, 0, 0, 0, 0, 0, 0, 0, 511, 535, 508, 526, 508, 500, 510, 512, 10, 4, 0, 0, 0, 0, 0, 0, 477, 475, 525, 502, 503, 514, 513, 468, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 498, 510, 523, 498, 512, 524, 24, 1, 0, 0, 0, 0, 0, 0, 522, 535, 510, 536, 506, 535, 498, 539, 0, 0, 0, 0, 0, 0, 0, 0, 522, 522, 500, 555, 528, 500, 474, 595, 0, 0, 0, 0, 0, 0, 0, 0, 521, 492, 515, 539, 510, 493, 522, 538, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 3986, 4124, 3977, 4088, 4181, 4196, 4138] avg=4096.0000 max=4196 max_rank=6 min=3977 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16603 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[471, 503, 552, 519, 496, 515, 537, 533, 0, 0, 0, 0, 0, 0, 0, 0, 534, 511, 525, 523, 518, 542, 516, 534, 0, 0, 0, 0, 0, 0, 0, 0, 495, 518, 539, 498, 476, 496, 522, 473, 11, 4, 0, 0, 0, 0, 0, 0, 544, 499, 518, 495, 507, 478, 518, 544, 0, 0, 0, 0, 0, 0, 0, 0, 512, 492, 516, 506, 506, 572, 518, 494, 25, 1, 0, 0, 0, 0, 0, 0, 485, 516, 516, 518, 519, 500, 500, 554, 0, 0, 0, 0, 0, 0, 0, 0, 511, 503, 519, 490, 553, 476, 494, 521, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 459, 527, 476, 501, 510, 501, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4203, 4032, 4103, 4142, 4108, 4067, 3987] avg=4096.0000 max=4203 max_rank=1 min=3987 min_rank=7 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 527, 521, 555, 532, 538, 479, 0, 0, 0, 0, 0, 0, 0, 0, 532, 502, 502, 520, 506, 512, 516, 491, 0, 0, 0, 0, 0, 0, 0, 0, 549, 536, 511, 510, 513, 474, 505, 527, 10, 4, 0, 0, 0, 0, 0, 0, 485, 532, 507, 520, 525, 508, 519, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 500, 521, 496, 498, 496, 527, 25, 1, 0, 0, 0, 0, 0, 0, 511, 496, 557, 501, 483, 535, 479, 512, 0, 0, 0, 0, 0, 0, 0, 0, 558, 496, 517, 512, 494, 476, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 499, 474, 494, 465, 513, 510, 544, 483, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4185, 4081, 4139, 4112, 4106, 4074, 4081, 3990] avg=4096.0000 max=4185 max_rank=0 min=3990 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16251 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 499, 523, 488, 506, 485, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 524, 479, 510, 503, 490, 501, 522, 492, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 500, 490, 502, 510, 482, 550, 15, 8, 0, 0, 0, 0, 0, 0, 556, 512, 487, 509, 537, 532, 546, 514, 0, 0, 0, 0, 0, 0, 0, 0, 540, 536, 491, 530, 486, 524, 508, 483, 28, 4, 0, 0, 0, 0, 0, 0, 511, 536, 499, 518, 526, 499, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 509, 457, 501, 509, 495, 513, 538, 526, 5, 0, 0, 0, 0, 0, 0, 0, 512, 534, 536, 541, 485, 541, 492, 535, 9, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4000, 4021, 4063, 4193, 4130, 4116, 4053, 4192] avg=4096.0000 max=4193 max_rank=3 min=4000 min_rank=0 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[494, 494, 503, 513, 539, 492, 482, 480, 512, 515, 478, 526, 514, 496, 516, 478, 530, 492, 563, 500, 513, 498, 510, 520, 537, 531, 518, 499, 527, 534, 559, 543, 486, 509, 484, 513, 518, 535, 536, 525, 499, 539, 492, 513, 481, 543, 537, 547, 551, 505, 536, 478, 521, 484, 527, 493, 511, 524, 500, 505, 493, 510, 481, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[456, 515, 551, 536, 484, 529, 523, 504, 528, 530, 520, 507, 549, 515, 478, 493, 528, 546, 501, 511, 511, 523, 505, 516, 485, 512, 553, 471, 539, 518, 479, 502, 528, 503, 539, 497, 530, 489, 469, 508, 521, 511, 476, 529, 524, 535, 481, 540, 485, 492, 502, 483, 470, 532, 499, 508, 521, 493, 505, 540, 552, 535, 534, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[557, 513, 469, 550, 494, 511, 556, 490, 534, 492, 512, 494, 506, 477, 538, 495, 500, 521, 503, 528, 543, 491, 489, 538, 480, 508, 512, 524, 517, 521, 535, 502, 528, 565, 481, 513, 503, 465, 507, 524, 543, 495, 547, 535, 489, 530, 515, 521, 516, 521, 500, 476, 523, 514, 515, 524, 534, 460, 529, 516, 497, 496, 487, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[531, 514, 564, 539, 490, 510, 510, 514, 507, 493, 502, 502, 508, 512, 492, 506, 506, 514, 539, 528, 499, 520, 525, 536, 503, 497, 505, 493, 476, 516, 496, 477, 520, 516, 505, 534, 522, 523, 517, 555, 536, 529, 481, 520, 521, 486, 488, 509, 521, 495, 539, 516, 538, 529, 488, 520, 508, 511, 537, 480, 520, 511, 477, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[506, 557, 497, 554, 520, 528, 499, 529, 520, 466, 500, 505, 520, 512, 525, 502, 503, 527, 493, 533, 559, 519, 516, 464, 503, 540, 472, 495, 547, 541, 553, 520, 505, 492, 497, 559, 508, 506, 521, 528, 495, 500, 504, 529, 521, 491, 514, 524, 510, 513, 481, 499, 513, 515, 531, 519, 509, 488, 495, 464, 487, 491, 520, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[500, 531, 499, 523, 515, 532, 530, 545, 492, 515, 501, 481, 498, 505, 494, 484, 556, 505, 542, 490, 504, 474, 480, 563, 455, 544, 503, 497, 518, 523, 533, 514, 518, 516, 503, 527, 517, 488, 515, 493, 526, 483, 514, 490, 526, 528, 530, 526, 518, 515, 513, 511, 541, 517, 484, 490, 523, 504, 509, 558, 509, 504, 490, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[528, 533, 496, 531, 504, 497, 506, 483, 537, 504, 532, 504, 491, 523, 483, 497, 501, 505, 536, 543, 502, 473, 526, 498, 511, 546, 480, 521, 521, 517, 476, 480, 523, 532, 549, 509, 516, 524, 494, 551, 494, 542, 518, 514, 486, 513, 531, 504, 553, 503, 514, 484, 501, 485, 510, 510, 522, 490, 543, 492, 522, 508, 516, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[527, 504, 533, 529, 534, 523, 507, 527, 533, 521, 521, 507, 533, 482, 546, 505, 503, 529, 525, 506, 485, 484, 512, 496, 494, 498, 502, 499, 502, 503, 526, 529, 505, 504, 543, 453, 544, 479, 505, 523, 527, 514, 530, 503, 510, 503, 474, 508, 524, 481, 470, 518, 505, 550, 533, 525, 525, 514, 528, 500, 498, 485, 521, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[456, 515, 551, 503, 484, 529, 523, 504, 0, 0, 0, 0, 0, 0, 0, 0, 528, 530, 520, 507, 549, 515, 478, 493, 33, 4, 0, 0, 0, 0, 0, 0, 528, 546, 486, 511, 511, 523, 505, 516, 0, 0, 0, 0, 0, 0, 0, 0, 485, 512, 553, 471, 539, 518, 479, 502, 4, 0, 0, 0, 0, 0, 0, 0, 528, 503, 539, 497, 530, 489, 469, 493, 0, 0, 0, 0, 0, 0, 0, 0, 521, 511, 476, 529, 524, 535, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 485, 492, 502, 483, 470, 532, 499, 508, 10, 6, 0, 0, 0, 0, 0, 0, 521, 493, 505, 540, 552, 535, 534, 519, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4157, 4126, 4063, 4048, 4103, 3987, 4219] avg=4096.0000 max=4219 max_rank=7 min=3987 min_rank=6 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[494, 494, 503, 482, 539, 492, 482, 480, 0, 0, 0, 0, 0, 0, 0, 0, 512, 515, 478, 526, 514, 496, 516, 478, 31, 5, 0, 0, 0, 0, 0, 0, 530, 492, 546, 500, 513, 498, 510, 520, 0, 0, 0, 0, 0, 0, 0, 0, 537, 531, 518, 499, 527, 534, 559, 543, 4, 0, 0, 0, 0, 0, 0, 0, 486, 509, 484, 513, 518, 535, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 499, 539, 492, 513, 481, 543, 537, 533, 0, 0, 0, 0, 0, 0, 0, 0, 551, 505, 536, 478, 521, 484, 527, 493, 10, 6, 0, 0, 0, 0, 0, 0, 511, 524, 500, 505, 493, 510, 481, 486, 12, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3966, 4071, 4109, 4252, 4091, 4137, 4111, 4031] avg=4096.0000 max=4252 max_rank=3 min=3966 min_rank=0 imbalance_ratio=1.038086 max_load_reduction=28516 max_load_reduction_pct=0.870239 moved_to_echo=16370 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 557, 497, 520, 520, 528, 499, 529, 0, 0, 0, 0, 0, 0, 0, 0, 520, 466, 500, 505, 520, 512, 525, 502, 34, 4, 0, 0, 0, 0, 0, 0, 503, 527, 479, 533, 559, 519, 516, 464, 0, 0, 0, 0, 0, 0, 0, 0, 503, 540, 472, 495, 547, 541, 553, 520, 4, 0, 0, 0, 0, 0, 0, 0, 505, 492, 497, 559, 508, 506, 521, 513, 0, 0, 0, 0, 0, 0, 0, 0, 495, 500, 504, 529, 521, 491, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 510, 513, 481, 499, 513, 515, 531, 519, 10, 6, 0, 0, 0, 0, 0, 0, 509, 488, 495, 464, 487, 491, 520, 514, 10, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4088, 4100, 4175, 4101, 4064, 4097, 3987] avg=4096.0000 max=4175 max_rank=3 min=3987 min_rank=7 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16249 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[557, 513, 469, 516, 494, 511, 556, 490, 0, 0, 0, 0, 0, 0, 0, 0, 534, 492, 512, 494, 506, 477, 538, 495, 34, 4, 0, 0, 0, 0, 0, 0, 500, 521, 488, 528, 543, 491, 489, 538, 0, 0, 0, 0, 0, 0, 0, 0, 480, 508, 512, 524, 517, 521, 535, 502, 4, 0, 0, 0, 0, 0, 0, 0, 528, 565, 481, 513, 503, 465, 507, 509, 0, 0, 0, 0, 0, 0, 0, 0, 543, 495, 547, 535, 489, 530, 515, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 521, 500, 476, 523, 514, 515, 524, 10, 6, 0, 0, 0, 0, 0, 0, 534, 460, 529, 516, 497, 496, 487, 499, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4086, 4098, 4103, 4071, 4161, 4105, 4038] avg=4096.0000 max=4161 max_rank=5 min=4038 min_rank=7 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16375 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[531, 514, 564, 506, 490, 510, 510, 514, 0, 0, 0, 0, 0, 0, 0, 0, 507, 493, 502, 502, 508, 512, 492, 506, 33, 4, 0, 0, 0, 0, 0, 0, 506, 514, 524, 528, 499, 520, 525, 536, 0, 0, 0, 0, 0, 0, 0, 0, 503, 497, 505, 493, 476, 516, 496, 477, 4, 0, 0, 0, 0, 0, 0, 0, 520, 516, 505, 534, 522, 523, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 481, 520, 521, 486, 488, 496, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 539, 516, 538, 529, 488, 520, 9, 6, 0, 0, 0, 0, 0, 0, 508, 511, 537, 480, 520, 511, 477, 492, 11, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4059, 4152, 3967, 4176, 4057, 4161, 4057] avg=4096.0000 max=4176 max_rank=4 min=3967 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16451 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[528, 533, 496, 498, 504, 497, 506, 483, 0, 0, 0, 0, 0, 0, 0, 0, 537, 504, 532, 504, 491, 523, 483, 497, 33, 4, 0, 0, 0, 0, 0, 0, 501, 505, 521, 543, 502, 473, 526, 498, 0, 0, 0, 0, 0, 0, 0, 0, 511, 546, 480, 521, 521, 517, 476, 480, 4, 0, 0, 0, 0, 0, 0, 0, 523, 532, 549, 509, 516, 524, 494, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 518, 514, 486, 513, 531, 491, 0, 0, 0, 0, 0, 0, 0, 0, 553, 503, 514, 484, 501, 485, 510, 510, 9, 6, 0, 0, 0, 0, 0, 0, 522, 490, 543, 492, 522, 508, 516, 530, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4108, 4069, 4056, 4183, 4089, 4075, 4143] avg=4096.0000 max=4183 max_rank=4 min=4045 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16490 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[500, 531, 499, 487, 515, 532, 530, 545, 0, 0, 0, 0, 0, 0, 0, 0, 492, 515, 501, 481, 498, 505, 494, 484, 36, 9, 0, 0, 0, 0, 0, 0, 556, 505, 517, 490, 504, 474, 480, 563, 0, 0, 0, 0, 0, 0, 0, 0, 455, 544, 503, 497, 518, 523, 533, 514, 8, 0, 0, 0, 0, 0, 0, 0, 518, 516, 503, 527, 517, 488, 515, 473, 0, 0, 0, 0, 0, 0, 0, 0, 526, 483, 514, 490, 526, 528, 530, 504, 0, 0, 0, 0, 0, 0, 0, 0, 518, 515, 513, 511, 541, 517, 484, 490, 14, 8, 0, 0, 0, 0, 0, 0, 523, 504, 509, 558, 509, 504, 490, 536, 16, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4015, 4089, 4095, 4057, 4101, 4111, 4161] avg=4096.0000 max=4161 max_rank=7 min=4015 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[527, 504, 533, 497, 534, 523, 507, 527, 0, 0, 0, 0, 0, 0, 0, 0, 533, 521, 521, 507, 533, 482, 546, 505, 32, 4, 0, 0, 0, 0, 0, 0, 503, 529, 510, 506, 485, 484, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 494, 498, 502, 499, 502, 503, 526, 529, 4, 0, 0, 0, 0, 0, 0, 0, 505, 504, 543, 453, 544, 479, 505, 508, 0, 0, 0, 0, 0, 0, 0, 0, 527, 514, 530, 503, 510, 503, 474, 495, 0, 0, 0, 0, 0, 0, 0, 0, 524, 481, 470, 518, 505, 550, 533, 525, 9, 6, 0, 0, 0, 0, 0, 0, 525, 514, 528, 500, 498, 485, 521, 541, 11, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4184, 4025, 4057, 4041, 4056, 4121, 4132] avg=4096.0000 max=4184 max_rank=1 min=4025 min_rank=2 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 3968, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 3968, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[524, 522, 516, 464, 522, 498, 522, 528, 543, 514, 480, 541, 500, 510, 489, 553, 508, 525, 502, 500, 507, 494, 530, 505, 506, 497, 512, 518, 520, 547, 487, 517, 537, 522, 527, 527, 481, 503, 507, 491, 481, 521, 527, 481, 513, 515, 522, 506, 509, 461, 557, 509, 536, 496, 529, 511, 518, 513, 502, 515, 560, 517, 484, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[527, 520, 505, 533, 530, 518, 483, 505, 470, 523, 490, 491, 516, 481, 563, 528, 545, 477, 502, 486, 512, 504, 493, 536, 491, 535, 547, 509, 518, 517, 525, 484, 494, 530, 498, 505, 521, 539, 474, 517, 507, 493, 513, 464, 545, 529, 518, 537, 487, 517, 516, 517, 510, 525, 509, 475, 487, 515, 534, 582, 497, 490, 537, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[472, 472, 538, 505, 502, 517, 543, 517, 541, 494, 523, 537, 519, 524, 505, 530, 513, 511, 517, 503, 531, 499, 507, 513, 538, 520, 528, 500, 487, 497, 546, 492, 507, 510, 516, 532, 502, 503, 496, 510, 525, 537, 493, 523, 530, 477, 542, 519, 487, 491, 497, 508, 502, 530, 482, 563, 518, 486, 530, 489, 508, 499, 559, 456] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[498, 530, 476, 532, 545, 495, 518, 537, 480, 517, 521, 560, 478, 523, 524, 515, 503, 586, 556, 539, 506, 498, 487, 510, 511, 500, 490, 506, 511, 487, 509, 523, 505, 508, 493, 521, 526, 546, 500, 535, 501, 525, 499, 501, 506, 484, 521, 500, 508, 490, 478, 476, 512, 522, 525, 543, 498, 515, 491, 514, 508, 525, 530, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[486, 500, 517, 541, 494, 452, 538, 557, 524, 532, 532, 482, 499, 511, 518, 508, 561, 555, 473, 534, 495, 504, 529, 521, 488, 493, 520, 470, 513, 523, 512, 516, 533, 504, 509, 508, 515, 480, 534, 533, 499, 504, 533, 541, 495, 511, 512, 477, 462, 534, 493, 542, 500, 515, 523, 523, 483, 498, 485, 567, 490, 541, 523, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[509, 504, 483, 542, 523, 505, 526, 523, 490, 536, 513, 486, 526, 541, 542, 481, 545, 523, 545, 508, 493, 528, 528, 545, 532, 526, 503, 487, 537, 500, 529, 478, 514, 513, 510, 506, 517, 495, 486, 500, 509, 530, 530, 489, 491, 540, 512, 473, 493, 510, 498, 538, 507, 535, 513, 527, 534, 475, 514, 486, 484, 489, 504, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[494, 528, 513, 495, 505, 528, 508, 525, 489, 473, 525, 541, 488, 507, 486, 462, 516, 508, 544, 508, 526, 516, 528, 560, 544, 498, 475, 498, 518, 537, 499, 534, 543, 553, 506, 522, 481, 538, 499, 516, 482, 508, 479, 493, 530, 521, 511, 527, 474, 490, 492, 490, 522, 538, 498, 527, 570, 486, 498, 521, 532, 518, 501, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[517, 504, 510, 548, 518, 499, 519, 532, 494, 496, 514, 558, 514, 507, 503, 499, 510, 571, 498, 531, 522, 525, 503, 506, 482, 490, 509, 533, 482, 520, 504, 485, 510, 490, 532, 501, 540, 497, 506, 495, 520, 551, 461, 474, 535, 536, 499, 521, 493, 515, 490, 494, 518, 549, 519, 492, 486, 516, 510, 520, 519, 507, 531, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[472, 472, 538, 505, 502, 517, 543, 508, 0, 0, 0, 0, 0, 0, 0, 0, 541, 494, 523, 537, 519, 524, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 513, 465, 517, 503, 531, 499, 507, 513, 0, 0, 0, 0, 0, 0, 0, 0, 538, 520, 528, 500, 487, 497, 546, 492, 18, 0, 0, 0, 0, 0, 0, 0, 507, 510, 516, 532, 502, 503, 496, 510, 0, 0, 0, 0, 0, 0, 0, 0, 525, 537, 493, 523, 530, 477, 542, 519, 7, 6, 0, 0, 0, 0, 0, 0, 487, 491, 497, 508, 502, 530, 482, 563, 21, 0, 0, 0, 0, 0, 0, 0, 518, 486, 530, 489, 508, 499, 559, 456, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4057, 4173, 4048, 4126, 4076, 4159, 4081, 4048] avg=4096.0000 max=4173 max_rank=1 min=4048 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[524, 522, 516, 464, 522, 498, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 543, 514, 480, 541, 500, 510, 489, 553, 0, 0, 0, 0, 0, 0, 0, 0, 508, 478, 502, 500, 507, 494, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 506, 497, 512, 518, 520, 547, 487, 517, 19, 0, 0, 0, 0, 0, 0, 0, 537, 522, 527, 527, 481, 503, 507, 491, 0, 0, 0, 0, 0, 0, 0, 0, 481, 521, 527, 481, 513, 515, 522, 506, 7, 7, 0, 0, 0, 0, 0, 0, 509, 461, 557, 509, 536, 496, 529, 511, 21, 0, 0, 0, 0, 0, 0, 0, 518, 513, 502, 515, 560, 517, 484, 489, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4130, 4024, 4123, 4095, 4080, 4129, 4101] avg=4096.0000 max=4130 max_rank=1 min=4024 min_rank=2 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16405 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[527, 520, 505, 533, 530, 518, 483, 497, 0, 0, 0, 0, 0, 0, 0, 0, 470, 523, 490, 491, 516, 481, 563, 528, 0, 0, 0, 0, 0, 0, 0, 0, 545, 435, 502, 486, 512, 504, 493, 536, 0, 0, 0, 0, 0, 0, 0, 0, 491, 535, 547, 509, 518, 517, 525, 484, 17, 0, 0, 0, 0, 0, 0, 0, 494, 530, 498, 505, 521, 539, 474, 517, 0, 0, 0, 0, 0, 0, 0, 0, 507, 493, 513, 464, 545, 529, 518, 537, 6, 6, 0, 0, 0, 0, 0, 0, 487, 517, 516, 517, 510, 525, 509, 475, 19, 0, 0, 0, 0, 0, 0, 0, 487, 515, 534, 582, 497, 490, 537, 522, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4062, 4013, 4143, 4078, 4118, 4075, 4166] avg=4096.0000 max=4166 max_rank=7 min=4013 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 530, 476, 532, 545, 495, 518, 527, 0, 0, 0, 0, 0, 0, 0, 0, 480, 517, 521, 560, 478, 523, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 533, 556, 539, 506, 498, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 490, 506, 511, 487, 509, 523, 21, 0, 0, 0, 0, 0, 0, 0, 505, 508, 493, 521, 526, 546, 500, 535, 0, 0, 0, 0, 0, 0, 0, 0, 501, 525, 499, 501, 506, 484, 521, 500, 8, 7, 0, 0, 0, 0, 0, 0, 508, 490, 478, 476, 512, 522, 525, 543, 24, 0, 0, 0, 0, 0, 0, 0, 498, 515, 491, 514, 508, 525, 530, 491, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4118, 4132, 4058, 4134, 4052, 4078, 4075] avg=4096.0000 max=4134 max_rank=4 min=4052 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16339 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[486, 500, 517, 541, 494, 452, 538, 547, 0, 0, 0, 0, 0, 0, 0, 0, 524, 532, 532, 482, 499, 511, 518, 508, 0, 0, 0, 0, 0, 0, 0, 0, 561, 505, 473, 534, 495, 504, 529, 521, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 520, 470, 513, 523, 512, 516, 20, 0, 0, 0, 0, 0, 0, 0, 533, 504, 509, 508, 515, 480, 534, 533, 0, 0, 0, 0, 0, 0, 0, 0, 499, 504, 533, 541, 495, 511, 512, 477, 8, 7, 0, 0, 0, 0, 0, 0, 462, 534, 493, 542, 500, 515, 523, 523, 22, 0, 0, 0, 0, 0, 0, 0, 483, 498, 485, 567, 490, 541, 523, 503, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4106, 4122, 4055, 4116, 4087, 4114, 4093] avg=4096.0000 max=4122 max_rank=2 min=4055 min_rank=3 imbalance_ratio=1.006348 max_load_reduction=28646 max_load_reduction_pct=0.874207 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 504, 483, 542, 523, 505, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 513, 486, 526, 541, 542, 481, 0, 0, 0, 0, 0, 0, 0, 0, 545, 476, 545, 508, 493, 528, 528, 545, 0, 0, 0, 0, 0, 0, 0, 0, 532, 526, 503, 487, 537, 500, 529, 478, 19, 0, 0, 0, 0, 0, 0, 0, 514, 513, 510, 506, 517, 495, 486, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 530, 530, 489, 491, 540, 512, 473, 7, 7, 0, 0, 0, 0, 0, 0, 493, 510, 498, 538, 507, 535, 513, 527, 21, 0, 0, 0, 0, 0, 0, 0, 534, 475, 514, 486, 484, 489, 504, 509, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4115, 4168, 4111, 4041, 4088, 4142, 3998] avg=4096.0000 max=4168 max_rank=2 min=3998 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16269 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[494, 528, 513, 495, 505, 528, 508, 515, 0, 0, 0, 0, 0, 0, 0, 0, 489, 473, 525, 541, 488, 507, 486, 462, 0, 0, 0, 0, 0, 0, 0, 0, 516, 462, 544, 508, 526, 516, 528, 560, 0, 0, 0, 0, 0, 0, 0, 0, 544, 498, 475, 498, 518, 537, 499, 534, 18, 0, 0, 0, 0, 0, 0, 0, 543, 553, 506, 522, 481, 538, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 479, 493, 530, 521, 511, 527, 7, 7, 0, 0, 0, 0, 0, 0, 474, 490, 492, 490, 522, 538, 498, 527, 21, 0, 0, 0, 0, 0, 0, 0, 570, 486, 498, 521, 532, 518, 501, 526, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 3971, 4160, 4121, 4158, 4065, 4052, 4155] avg=4096.0000 max=4160 max_rank=2 min=3971 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[517, 504, 510, 548, 518, 499, 519, 510, 0, 0, 0, 0, 0, 0, 0, 0, 494, 496, 514, 558, 514, 507, 503, 499, 7, 1, 0, 0, 0, 0, 0, 0, 510, 506, 498, 531, 522, 525, 503, 506, 0, 0, 0, 0, 0, 0, 0, 0, 482, 490, 509, 533, 482, 520, 504, 485, 26, 0, 0, 0, 0, 0, 0, 0, 509, 490, 532, 501, 540, 497, 506, 495, 0, 0, 0, 0, 0, 0, 0, 0, 520, 551, 461, 474, 535, 536, 499, 521, 12, 10, 0, 0, 0, 0, 0, 0, 493, 515, 490, 494, 518, 549, 519, 492, 27, 0, 0, 0, 0, 0, 0, 0, 486, 516, 510, 520, 519, 507, 531, 538, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4093, 4101, 4031, 4070, 4119, 4097, 4132] avg=4096.0000 max=4132 max_rank=7 min=4031 min_rank=3 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4096, 4096, 4224, 4224, 4224, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[518, 505, 473, 543, 502, 492, 529, 490, 548, 512, 509, 501, 519, 516, 505, 509, 500, 521, 498, 500, 500, 511, 505, 507, 478, 524, 525, 514, 537, 524, 539, 511, 532, 508, 516, 523, 517, 512, 469, 511, 513, 500, 503, 540, 523, 531, 517, 486, 499, 528, 524, 485, 515, 540, 505, 442, 532, 524, 524, 500, 535, 556, 495, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[487, 525, 535, 533, 516, 517, 501, 511, 526, 533, 523, 496, 474, 489, 535, 553, 511, 508, 499, 520, 516, 521, 518, 507, 489, 498, 502, 540, 529, 500, 527, 461, 554, 512, 475, 546, 500, 519, 529, 505, 546, 515, 480, 485, 523, 529, 526, 496, 510, 528, 505, 502, 508, 525, 505, 538, 455, 523, 488, 559, 504, 509, 463, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[485, 519, 518, 525, 516, 512, 480, 514, 525, 530, 509, 538, 508, 520, 524, 499, 545, 493, 518, 509, 517, 497, 472, 503, 470, 526, 482, 486, 564, 497, 520, 538, 513, 506, 504, 508, 514, 509, 504, 492, 531, 529, 531, 514, 565, 499, 487, 546, 545, 496, 496, 527, 494, 475, 506, 501, 524, 474, 520, 520, 493, 534, 513, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[541, 511, 494, 550, 518, 518, 503, 539, 510, 528, 522, 525, 491, 489, 527, 529, 497, 492, 528, 460, 538, 472, 534, 512, 515, 535, 525, 513, 486, 511, 514, 498, 505, 506, 517, 492, 509, 491, 526, 534, 484, 526, 542, 510, 529, 515, 521, 499, 525, 492, 492, 525, 501, 516, 515, 528, 512, 510, 489, 485, 518, 521, 506, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[519, 531, 520, 534, 516, 512, 500, 510, 498, 541, 515, 512, 531, 530, 513, 489, 492, 518, 516, 516, 486, 483, 515, 505, 486, 516, 513, 579, 441, 508, 528, 525, 530, 544, 436, 522, 508, 519, 536, 514, 528, 458, 511, 510, 522, 482, 492, 523, 520, 485, 518, 495, 506, 507, 546, 509, 467, 539, 548, 520, 517, 528, 528, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[511, 529, 498, 547, 501, 487, 494, 479, 519, 528, 497, 541, 517, 523, 490, 537, 534, 508, 548, 512, 482, 507, 506, 484, 533, 531, 540, 504, 535, 490, 488, 517, 537, 497, 482, 551, 514, 520, 550, 525, 506, 479, 514, 495, 502, 533, 479, 509, 500, 526, 483, 516, 498, 504, 531, 505, 473, 545, 493, 543, 525, 498, 506, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[578, 491, 551, 483, 526, 527, 543, 533, 496, 511, 507, 483, 495, 506, 531, 482, 477, 509, 513, 480, 533, 509, 517, 543, 563, 529, 511, 503, 536, 480, 460, 519, 529, 515, 519, 524, 518, 493, 550, 503, 493, 486, 490, 511, 505, 494, 517, 482, 484, 496, 511, 538, 506, 494, 517, 536, 514, 474, 509, 534, 521, 523, 511, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[532, 533, 539, 493, 529, 506, 513, 523, 511, 528, 544, 492, 517, 495, 554, 505, 534, 466, 533, 491, 529, 499, 485, 535, 511, 510, 492, 516, 512, 516, 527, 546, 512, 485, 487, 474, 526, 528, 486, 499, 499, 457, 550, 476, 522, 501, 485, 510, 500, 510, 531, 515, 549, 516, 493, 503, 500, 529, 577, 496, 478, 483, 547, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[518, 505, 473, 513, 502, 492, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 548, 489, 509, 501, 519, 516, 505, 509, 0, 0, 0, 0, 0, 0, 0, 0, 500, 521, 498, 500, 500, 511, 505, 507, 30, 4, 0, 0, 0, 0, 0, 0, 478, 515, 525, 514, 537, 524, 539, 511, 0, 0, 0, 0, 0, 0, 0, 0, 530, 508, 516, 523, 517, 512, 469, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 503, 540, 523, 531, 517, 486, 19, 5, 0, 0, 0, 0, 0, 0, 499, 528, 524, 485, 515, 540, 505, 442, 9, 3, 2, 0, 0, 0, 0, 0, 532, 524, 524, 492, 535, 556, 495, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4096, 4076, 4143, 4086, 4137, 4052, 4156] avg=4096.0000 max=4156 max_rank=7 min=4022 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[487, 525, 535, 503, 516, 517, 501, 511, 0, 0, 0, 0, 0, 0, 0, 0, 526, 510, 523, 496, 474, 489, 535, 553, 0, 0, 0, 0, 0, 0, 0, 0, 511, 508, 499, 520, 516, 521, 518, 507, 30, 4, 0, 0, 0, 0, 0, 0, 489, 490, 502, 540, 529, 500, 527, 461, 0, 0, 0, 0, 0, 0, 0, 0, 551, 512, 475, 546, 500, 519, 529, 505, 0, 0, 0, 0, 0, 0, 0, 0, 546, 515, 480, 485, 523, 529, 526, 496, 19, 6, 0, 0, 0, 0, 0, 0, 510, 528, 505, 502, 508, 525, 505, 538, 8, 3, 3, 0, 0, 0, 0, 0, 455, 523, 488, 550, 504, 509, 463, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4106, 4134, 4038, 4137, 4125, 4135, 3998] avg=4096.0000 max=4137 max_rank=4 min=3998 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[541, 511, 494, 519, 518, 518, 503, 539, 0, 0, 0, 0, 0, 0, 0, 0, 510, 505, 522, 525, 491, 489, 527, 529, 0, 0, 0, 0, 0, 0, 0, 0, 497, 492, 528, 460, 538, 472, 534, 512, 31, 4, 0, 0, 0, 0, 0, 0, 515, 526, 525, 513, 486, 511, 514, 498, 0, 0, 0, 0, 0, 0, 0, 0, 503, 506, 517, 492, 509, 491, 526, 534, 0, 0, 0, 0, 0, 0, 0, 0, 484, 526, 542, 510, 529, 515, 521, 499, 19, 5, 0, 0, 0, 0, 0, 0, 525, 492, 492, 525, 501, 516, 515, 528, 9, 3, 2, 0, 0, 0, 0, 0, 512, 510, 489, 477, 518, 521, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4098, 4068, 4088, 4078, 4150, 4108, 4035] avg=4096.0000 max=4150 max_rank=5 min=4035 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 529, 498, 516, 501, 487, 494, 479, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 497, 541, 517, 523, 490, 537, 0, 0, 0, 0, 0, 0, 0, 0, 534, 508, 548, 512, 482, 507, 506, 484, 31, 4, 0, 0, 0, 0, 0, 0, 533, 522, 540, 504, 535, 490, 488, 517, 0, 0, 0, 0, 0, 0, 0, 0, 535, 497, 482, 551, 514, 520, 550, 525, 0, 0, 0, 0, 0, 0, 0, 0, 506, 479, 514, 495, 502, 533, 479, 509, 19, 6, 0, 0, 0, 0, 0, 0, 500, 526, 483, 516, 498, 504, 531, 505, 9, 3, 2, 0, 0, 0, 0, 0, 473, 545, 493, 534, 525, 498, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4129, 4116, 4129, 4174, 4042, 4077, 4086] avg=4096.0000 max=4174 max_rank=4 min=4015 min_rank=0 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 531, 520, 504, 516, 512, 500, 510, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 515, 512, 531, 530, 513, 489, 0, 0, 0, 0, 0, 0, 0, 0, 492, 518, 516, 516, 486, 483, 515, 505, 30, 4, 0, 0, 0, 0, 0, 0, 486, 507, 513, 579, 441, 508, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 528, 544, 436, 522, 508, 519, 536, 514, 0, 0, 0, 0, 0, 0, 0, 0, 528, 458, 511, 510, 522, 482, 492, 523, 20, 6, 0, 0, 0, 0, 0, 0, 520, 485, 518, 495, 506, 507, 546, 509, 9, 3, 2, 0, 0, 0, 0, 0, 467, 539, 548, 511, 517, 528, 528, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4105, 4065, 4087, 4107, 4052, 4100, 4140] avg=4096.0000 max=4140 max_rank=7 min=4052 min_rank=5 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 533, 539, 465, 529, 506, 513, 523, 0, 0, 0, 0, 0, 0, 0, 0, 511, 505, 544, 492, 517, 495, 554, 505, 0, 0, 0, 0, 0, 0, 0, 0, 534, 466, 533, 491, 529, 499, 485, 535, 28, 4, 0, 0, 0, 0, 0, 0, 511, 501, 492, 516, 512, 516, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0, 510, 485, 487, 474, 526, 528, 486, 499, 0, 0, 0, 0, 0, 0, 0, 0, 499, 457, 550, 476, 522, 501, 485, 510, 19, 5, 0, 0, 0, 0, 0, 0, 500, 510, 531, 515, 549, 516, 493, 503, 9, 3, 2, 0, 0, 0, 0, 0, 500, 529, 577, 488, 478, 483, 547, 528, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4123, 4104, 4121, 3995, 4024, 4131, 4130] avg=4096.0000 max=4140 max_rank=0 min=3995 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[578, 491, 551, 456, 526, 527, 543, 533, 0, 0, 0, 0, 0, 0, 0, 0, 496, 488, 507, 483, 495, 506, 531, 482, 0, 0, 0, 0, 0, 0, 0, 0, 477, 509, 513, 480, 533, 509, 517, 543, 27, 4, 0, 0, 0, 0, 0, 0, 563, 520, 511, 503, 536, 480, 460, 519, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 519, 524, 518, 493, 550, 503, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 490, 511, 505, 494, 517, 482, 19, 6, 0, 0, 0, 0, 0, 0, 484, 496, 511, 538, 506, 494, 517, 536, 9, 3, 2, 0, 0, 0, 0, 0, 514, 474, 509, 525, 521, 523, 511, 546, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4205, 3988, 4112, 4092, 4149, 4003, 4096, 4123] avg=4096.0000 max=4205 max_rank=0 min=3988 min_rank=1 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[485, 519, 518, 492, 516, 512, 480, 514, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 509, 538, 508, 520, 524, 499, 0, 0, 0, 0, 0, 0, 0, 0, 545, 493, 518, 509, 517, 497, 472, 503, 33, 6, 0, 0, 0, 0, 0, 0, 470, 513, 482, 486, 564, 497, 520, 538, 0, 0, 0, 0, 0, 0, 0, 0, 505, 506, 504, 508, 514, 509, 504, 492, 0, 0, 0, 0, 0, 0, 0, 0, 531, 529, 531, 514, 565, 499, 487, 546, 24, 9, 0, 0, 0, 0, 0, 0, 545, 496, 496, 527, 494, 475, 506, 501, 13, 8, 8, 0, 0, 0, 0, 0, 524, 474, 520, 503, 493, 534, 513, 539, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4123, 4093, 4070, 4042, 4235, 4069, 4100] avg=4096.0000 max=4235 max_rank=5 min=4036 min_rank=0 imbalance_ratio=1.033936 max_load_reduction=28533 max_load_reduction_pct=0.870758 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[491, 487, 492, 518, 511, 533, 565, 484, 530, 519, 509, 538, 505, 487, 505, 516, 521, 496, 498, 521, 509, 532, 544, 502, 501, 488, 517, 507, 561, 495, 534, 510, 477, 529, 497, 489, 506, 492, 526, 546, 554, 483, 537, 510, 522, 501, 495, 553, 540, 537, 498, 533, 504, 518, 528, 511, 500, 519, 491, 513, 497, 515, 443, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[511, 528, 502, 545, 507, 527, 500, 502, 537, 499, 550, 521, 511, 508, 489, 491, 487, 519, 533, 517, 482, 516, 529, 482, 491, 536, 488, 495, 525, 507, 537, 478, 505, 533, 500, 537, 521, 494, 497, 531, 512, 540, 508, 526, 504, 524, 479, 523, 520, 486, 539, 491, 490, 517, 541, 491, 521, 509, 495, 513, 514, 506, 529, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[496, 487, 519, 463, 466, 502, 514, 503, 526, 493, 513, 508, 510, 537, 527, 514, 535, 509, 554, 495, 521, 494, 516, 519, 492, 506, 517, 502, 529, 511, 487, 516, 503, 514, 513, 522, 480, 528, 492, 503, 514, 510, 550, 506, 514, 537, 494, 544, 550, 501, 548, 509, 512, 495, 510, 513, 512, 524, 503, 555, 520, 528, 495, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[527, 512, 527, 538, 509, 512, 522, 503, 484, 507, 480, 480, 488, 545, 524, 517, 514, 503, 487, 503, 536, 523, 532, 525, 494, 537, 528, 540, 506, 522, 483, 495, 542, 498, 496, 488, 491, 507, 503, 488, 467, 497, 534, 516, 490, 550, 515, 508, 522, 513, 471, 495, 525, 508, 520, 524, 519, 539, 512, 526, 526, 539, 533, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[525, 509, 561, 477, 530, 495, 525, 525, 542, 527, 539, 497, 499, 545, 534, 500, 512, 496, 505, 494, 524, 529, 482, 487, 508, 499, 514, 501, 518, 493, 481, 509, 547, 480, 484, 517, 535, 526, 553, 511, 509, 508, 470, 499, 518, 506, 544, 544, 512, 523, 513, 494, 518, 494, 543, 536, 478, 512, 493, 512, 503, 492, 492, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[517, 531, 533, 536, 492, 485, 522, 510, 531, 508, 504, 519, 524, 506, 557, 506, 498, 513, 514, 535, 521, 523, 506, 488, 545, 506, 525, 497, 513, 514, 504, 518, 514, 482, 516, 479, 501, 514, 508, 508, 512, 485, 499, 518, 490, 493, 521, 546, 502, 520, 541, 485, 492, 497, 527, 535, 523, 508, 505, 534, 489, 498, 518, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[512, 537, 512, 513, 487, 500, 508, 485, 545, 525, 493, 506, 490, 541, 488, 530, 486, 509, 491, 539, 495, 528, 511, 547, 543, 488, 516, 541, 481, 499, 524, 489, 525, 527, 524, 520, 516, 513, 516, 529, 495, 550, 523, 466, 488, 480, 538, 492, 526, 552, 515, 511, 498, 526, 524, 500, 517, 478, 503, 491, 481, 516, 529, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[473, 488, 531, 529, 495, 530, 515, 519, 528, 515, 514, 531, 512, 510, 478, 485, 528, 537, 520, 504, 510, 523, 480, 494, 522, 510, 489, 508, 519, 498, 530, 541, 479, 530, 506, 515, 513, 516, 491, 514, 515, 504, 511, 502, 543, 488, 498, 557, 512, 491, 521, 530, 524, 520, 508, 530, 527, 503, 504, 508, 523, 503, 504, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[491, 487, 492, 518, 511, 533, 565, 484, 5, 1, 0, 0, 0, 0, 0, 0, 514, 519, 509, 538, 505, 487, 505, 516, 0, 0, 0, 0, 0, 0, 0, 0, 521, 496, 498, 521, 509, 531, 544, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 488, 517, 507, 561, 495, 534, 510, 9, 2, 0, 0, 0, 0, 0, 0, 477, 529, 497, 489, 506, 492, 526, 546, 7, 6, 0, 0, 0, 0, 0, 0, 554, 483, 537, 510, 522, 501, 495, 546, 0, 0, 0, 0, 0, 0, 0, 0, 540, 537, 498, 533, 504, 518, 503, 511, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 491, 513, 497, 515, 443, 478, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4093, 4122, 4124, 4075, 4148, 4144, 3975] avg=4096.0000 max=4148 max_rank=5 min=3975 min_rank=7 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 487, 519, 463, 466, 502, 514, 503, 5, 1, 0, 0, 0, 0, 0, 0, 511, 493, 513, 508, 510, 537, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 554, 495, 521, 493, 516, 519, 0, 0, 0, 0, 0, 0, 0, 0, 492, 506, 517, 502, 529, 511, 487, 516, 8, 2, 0, 0, 0, 0, 0, 0, 503, 514, 513, 522, 480, 528, 492, 503, 7, 6, 0, 0, 0, 0, 0, 0, 514, 510, 550, 506, 514, 537, 494, 537, 0, 0, 0, 0, 0, 0, 0, 0, 550, 501, 548, 509, 512, 495, 485, 513, 0, 0, 0, 0, 0, 0, 0, 0, 512, 524, 503, 555, 520, 528, 495, 488, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3956, 4113, 4142, 4070, 4068, 4162, 4113, 4144] avg=4096.0000 max=4162 max_rank=5 min=3956 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16487 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[525, 509, 561, 477, 530, 495, 525, 525, 5, 1, 0, 0, 0, 0, 0, 0, 526, 527, 539, 497, 499, 545, 534, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 496, 505, 494, 524, 528, 482, 487, 0, 0, 0, 0, 0, 0, 0, 0, 508, 499, 514, 501, 518, 493, 481, 509, 9, 2, 0, 0, 0, 0, 0, 0, 547, 480, 484, 517, 535, 526, 553, 511, 7, 6, 0, 0, 0, 0, 0, 0, 509, 508, 470, 499, 518, 506, 544, 537, 0, 0, 0, 0, 0, 0, 0, 0, 512, 523, 513, 494, 518, 494, 517, 536, 0, 0, 0, 0, 0, 0, 0, 0, 478, 512, 493, 512, 503, 492, 492, 520, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4167, 4028, 4034, 4166, 4091, 4107, 4022] avg=4096.0000 max=4167 max_rank=1 min=4022 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 528, 502, 545, 507, 527, 500, 502, 5, 1, 0, 0, 0, 0, 0, 0, 521, 499, 550, 521, 511, 508, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 487, 519, 533, 517, 482, 515, 529, 482, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 488, 495, 525, 507, 537, 478, 9, 2, 0, 0, 0, 0, 0, 0, 505, 533, 500, 537, 521, 494, 497, 531, 7, 6, 0, 0, 0, 0, 0, 0, 512, 540, 508, 526, 504, 524, 479, 516, 0, 0, 0, 0, 0, 0, 0, 0, 520, 486, 539, 491, 490, 517, 515, 491, 0, 0, 0, 0, 0, 0, 0, 0, 521, 509, 495, 513, 514, 506, 529, 522, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4090, 4064, 4068, 4131, 4109, 4049, 4129] avg=4096.0000 max=4131 max_rank=4 min=4049 min_rank=6 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[527, 512, 527, 538, 509, 512, 522, 503, 5, 1, 0, 0, 0, 0, 0, 0, 470, 507, 480, 480, 488, 545, 524, 517, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 487, 503, 536, 522, 532, 525, 0, 0, 0, 0, 0, 0, 0, 0, 494, 537, 528, 540, 506, 522, 483, 495, 8, 2, 0, 0, 0, 0, 0, 0, 542, 498, 496, 488, 491, 507, 503, 488, 6, 6, 0, 0, 0, 0, 0, 0, 467, 497, 534, 516, 490, 550, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 471, 495, 525, 508, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 519, 539, 512, 526, 526, 539, 533, 503, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4011, 4122, 4115, 4025, 4070, 4053, 4216] avg=4096.0000 max=4216 max_rank=7 min=4011 min_rank=1 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[473, 488, 531, 529, 495, 530, 515, 519, 5, 1, 0, 0, 0, 0, 0, 0, 512, 515, 514, 531, 512, 510, 478, 485, 0, 0, 0, 0, 0, 0, 0, 0, 528, 537, 520, 504, 510, 522, 480, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 510, 489, 508, 519, 498, 530, 541, 9, 2, 0, 0, 0, 0, 0, 0, 479, 530, 506, 515, 513, 516, 491, 514, 7, 6, 0, 0, 0, 0, 0, 0, 515, 504, 511, 502, 543, 488, 498, 550, 0, 0, 0, 0, 0, 0, 0, 0, 512, 491, 521, 530, 524, 520, 483, 530, 0, 0, 0, 0, 0, 0, 0, 0, 527, 503, 504, 508, 523, 503, 504, 512, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4057, 4095, 4128, 4077, 4111, 4111, 4103] avg=4096.0000 max=4128 max_rank=3 min=4057 min_rank=1 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 537, 512, 513, 487, 500, 508, 485, 4, 1, 0, 0, 0, 0, 0, 0, 529, 525, 493, 506, 490, 541, 488, 530, 0, 0, 0, 0, 0, 0, 0, 0, 486, 509, 491, 539, 495, 527, 511, 547, 0, 0, 0, 0, 0, 0, 0, 0, 543, 488, 516, 541, 481, 499, 524, 489, 9, 2, 0, 0, 0, 0, 0, 0, 525, 527, 524, 520, 516, 513, 516, 529, 7, 6, 0, 0, 0, 0, 0, 0, 495, 550, 523, 466, 488, 480, 538, 486, 0, 0, 0, 0, 0, 0, 0, 0, 526, 552, 515, 511, 498, 526, 499, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 478, 503, 491, 481, 516, 529, 540, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4102, 4105, 4092, 4183, 4026, 4127, 4074] avg=4096.0000 max=4183 max_rank=4 min=4026 min_rank=5 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16410 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[517, 531, 533, 536, 492, 485, 522, 510, 9, 8, 0, 0, 0, 0, 0, 0, 511, 508, 504, 519, 524, 506, 557, 506, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 514, 535, 521, 515, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0, 545, 506, 525, 497, 513, 514, 504, 518, 11, 4, 0, 0, 0, 0, 0, 0, 514, 482, 516, 479, 501, 514, 508, 508, 9, 12, 0, 0, 0, 0, 0, 0, 512, 485, 499, 518, 490, 493, 521, 533, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 541, 485, 492, 497, 492, 535, 0, 0, 0, 0, 0, 0, 0, 0, 523, 508, 505, 534, 489, 498, 518, 507, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4143, 4135, 4090, 4137, 4043, 4051, 4064, 4105] avg=4096.0000 max=4143 max_rank=0 min=4043 min_rank=4 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16263 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[477, 515, 507, 484, 528, 528, 533, 523, 547, 519, 497, 509, 519, 492, 522, 553, 520, 514, 498, 502, 515, 547, 530, 476, 493, 520, 538, 530, 502, 505, 532, 504, 467, 542, 513, 556, 542, 489, 505, 497, 496, 534, 554, 516, 511, 528, 519, 512, 490, 522, 459, 529, 517, 479, 492, 483, 537, 496, 499, 516, 495, 488, 518, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[493, 511, 537, 541, 519, 528, 525, 522, 489, 504, 517, 541, 476, 522, 515, 528, 522, 501, 508, 489, 523, 508, 553, 491, 490, 523, 546, 525, 502, 537, 532, 508, 523, 510, 504, 522, 527, 521, 503, 505, 533, 495, 483, 470, 538, 506, 518, 531, 520, 515, 502, 520, 521, 495, 486, 495, 512, 514, 513, 511, 469, 466, 491, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[518, 503, 488, 538, 521, 512, 491, 469, 507, 520, 458, 470, 532, 530, 547, 507, 475, 469, 543, 514, 534, 534, 499, 548, 527, 511, 506, 544, 533, 512, 513, 553, 507, 535, 509, 510, 536, 506, 516, 512, 531, 499, 510, 449, 512, 530, 519, 499, 509, 484, 503, 469, 480, 471, 542, 506, 528, 525, 515, 512, 566, 518, 526, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[473, 514, 530, 514, 518, 511, 496, 533, 541, 511, 504, 506, 551, 563, 508, 510, 504, 482, 507, 520, 495, 501, 504, 508, 492, 520, 501, 499, 545, 493, 510, 511, 515, 487, 511, 535, 481, 516, 496, 478, 470, 527, 504, 528, 568, 529, 552, 497, 509, 525, 503, 527, 529, 490, 546, 522, 523, 508, 517, 519, 517, 477, 470, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[549, 501, 514, 506, 497, 503, 532, 512, 507, 536, 521, 514, 541, 526, 500, 502, 548, 491, 546, 535, 532, 501, 505, 506, 504, 538, 502, 518, 500, 500, 521, 546, 541, 507, 510, 498, 517, 531, 517, 502, 535, 495, 517, 529, 513, 466, 550, 502, 467, 488, 497, 497, 500, 509, 456, 487, 518, 537, 512, 501, 494, 513, 507, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[522, 497, 539, 501, 521, 533, 490, 544, 493, 502, 501, 524, 474, 517, 538, 543, 534, 483, 479, 523, 501, 515, 504, 545, 527, 477, 558, 528, 531, 539, 512, 496, 515, 523, 502, 482, 502, 503, 508, 486, 529, 476, 538, 520, 519, 552, 525, 518, 494, 524, 501, 500, 518, 485, 540, 509, 456, 526, 504, 501, 478, 486, 508, 549] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[516, 480, 519, 535, 459, 528, 531, 477, 516, 507, 517, 503, 510, 535, 534, 535, 464, 533, 526, 519, 486, 497, 513, 534, 483, 535, 497, 528, 511, 506, 518, 502, 493, 482, 508, 507, 509, 536, 503, 481, 506, 524, 524, 503, 463, 513, 489, 529, 523, 507, 508, 522, 547, 544, 522, 498, 533, 543, 527, 486, 507, 518, 532, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[530, 536, 515, 503, 519, 541, 490, 507, 496, 543, 502, 481, 517, 517, 494, 519, 522, 559, 486, 500, 498, 512, 506, 547, 524, 482, 496, 522, 515, 505, 512, 507, 490, 530, 523, 538, 494, 469, 508, 518, 554, 536, 480, 496, 526, 525, 532, 537, 478, 535, 497, 529, 506, 500, 509, 522, 480, 520, 523, 525, 501, 487, 505, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[477, 515, 507, 484, 528, 519, 533, 523, 0, 0, 0, 0, 0, 0, 0, 0, 547, 519, 497, 509, 519, 468, 522, 553, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 498, 502, 515, 547, 530, 475, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 538, 502, 502, 505, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 467, 542, 513, 556, 542, 489, 505, 497, 9, 7, 1, 0, 0, 0, 0, 0, 496, 534, 554, 516, 511, 528, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 490, 522, 459, 529, 517, 479, 492, 483, 28, 22, 0, 0, 0, 0, 0, 0, 537, 496, 499, 516, 495, 488, 518, 488, 17, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4134, 4101, 4096, 4128, 4143, 4021, 4059] avg=4096.0000 max=4143 max_rank=5 min=4021 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[493, 511, 537, 541, 519, 519, 525, 522, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 517, 541, 476, 497, 515, 528, 0, 0, 0, 0, 0, 0, 0, 0, 522, 501, 508, 489, 523, 508, 553, 490, 0, 0, 0, 0, 0, 0, 0, 0, 490, 523, 546, 497, 502, 537, 532, 508, 0, 0, 0, 0, 0, 0, 0, 0, 523, 510, 504, 522, 527, 521, 503, 505, 9, 7, 1, 0, 0, 0, 0, 0, 533, 495, 483, 470, 538, 506, 491, 531, 0, 0, 0, 0, 0, 0, 0, 0, 520, 515, 502, 520, 521, 495, 486, 495, 28, 22, 0, 0, 0, 0, 0, 0, 512, 514, 513, 511, 469, 466, 491, 523, 18, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4167, 4067, 4094, 4135, 4132, 4047, 4104, 4022] avg=4096.0000 max=4167 max_rank=0 min=4022 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16305 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[518, 503, 488, 538, 521, 503, 491, 469, 0, 0, 0, 0, 0, 0, 0, 0, 507, 520, 458, 470, 532, 504, 547, 507, 0, 0, 0, 0, 0, 0, 0, 0, 475, 469, 543, 514, 534, 534, 499, 546, 0, 0, 0, 0, 0, 0, 0, 0, 527, 511, 506, 515, 533, 512, 513, 553, 0, 0, 0, 0, 0, 0, 0, 0, 507, 535, 509, 510, 536, 506, 516, 512, 9, 8, 2, 0, 0, 0, 0, 0, 531, 499, 510, 449, 512, 530, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 509, 484, 503, 469, 480, 471, 542, 506, 29, 22, 0, 0, 0, 0, 0, 0, 528, 525, 515, 512, 566, 518, 526, 508, 18, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4045, 4114, 4170, 4150, 4022, 4015, 4221] avg=4096.0000 max=4221 max_rank=7 min=4015 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[473, 514, 530, 514, 518, 502, 496, 533, 0, 0, 0, 0, 0, 0, 0, 0, 541, 511, 504, 506, 551, 536, 508, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 482, 507, 520, 495, 501, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 492, 520, 501, 472, 545, 493, 510, 511, 0, 0, 0, 0, 0, 0, 0, 0, 515, 487, 511, 535, 481, 516, 496, 478, 9, 8, 1, 0, 0, 0, 0, 0, 470, 527, 504, 528, 568, 529, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 509, 525, 503, 527, 529, 490, 546, 522, 27, 23, 0, 0, 0, 0, 0, 0, 523, 508, 517, 519, 517, 477, 470, 517, 19, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4167, 4020, 4044, 4037, 4147, 4201, 4072] avg=4096.0000 max=4201 max_rank=6 min=4020 min_rank=2 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[549, 501, 514, 506, 497, 494, 532, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 536, 521, 514, 541, 500, 500, 502, 0, 0, 0, 0, 0, 0, 0, 0, 548, 491, 546, 535, 532, 501, 505, 505, 0, 0, 0, 0, 0, 0, 0, 0, 504, 538, 502, 490, 500, 500, 521, 546, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 510, 498, 517, 531, 517, 502, 9, 8, 1, 0, 0, 0, 0, 0, 535, 495, 517, 529, 513, 466, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 467, 488, 497, 497, 500, 509, 456, 487, 28, 23, 0, 0, 0, 0, 0, 0, 518, 537, 512, 501, 494, 513, 507, 501, 18, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4121, 4163, 4101, 4141, 4079, 3952, 4106] avg=4096.0000 max=4163 max_rank=2 min=3952 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16278 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 536, 515, 503, 519, 531, 490, 507, 0, 0, 0, 0, 0, 0, 0, 0, 496, 543, 502, 481, 517, 492, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 559, 486, 500, 498, 512, 506, 545, 0, 0, 0, 0, 0, 0, 0, 0, 524, 482, 496, 494, 515, 505, 512, 507, 0, 0, 0, 0, 0, 0, 0, 0, 490, 530, 523, 538, 494, 469, 508, 518, 10, 7, 2, 0, 0, 0, 0, 0, 554, 536, 480, 496, 526, 525, 505, 537, 0, 0, 0, 0, 0, 0, 0, 0, 478, 535, 497, 529, 506, 500, 509, 522, 28, 22, 0, 0, 0, 0, 0, 0, 480, 520, 523, 525, 501, 487, 505, 492, 18, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4044, 4128, 4035, 4089, 4159, 4126, 4056] avg=4096.0000 max=4159 max_rank=5 min=4035 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 480, 519, 535, 459, 519, 531, 477, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 517, 503, 510, 509, 534, 535, 0, 0, 0, 0, 0, 0, 0, 0, 464, 533, 526, 519, 486, 497, 513, 532, 0, 0, 0, 0, 0, 0, 0, 0, 483, 535, 497, 500, 511, 506, 518, 502, 0, 0, 0, 0, 0, 0, 0, 0, 493, 482, 508, 507, 509, 536, 503, 481, 9, 8, 2, 0, 0, 0, 0, 0, 506, 524, 524, 503, 463, 513, 465, 529, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 508, 522, 547, 544, 522, 498, 28, 20, 0, 0, 0, 0, 0, 0, 533, 543, 527, 486, 507, 518, 532, 527, 18, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4131, 4070, 4052, 4038, 4027, 4219, 4195] avg=4096.0000 max=4219 max_rank=6 min=4027 min_rank=5 imbalance_ratio=1.030029 max_load_reduction=28549 max_load_reduction_pct=0.871246 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 497, 539, 501, 521, 518, 490, 544, 0, 0, 0, 0, 0, 0, 0, 0, 493, 502, 501, 524, 474, 484, 538, 543, 0, 0, 0, 0, 0, 0, 0, 0, 534, 483, 479, 523, 501, 515, 504, 539, 0, 0, 0, 0, 0, 0, 0, 0, 527, 477, 558, 495, 531, 539, 512, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 502, 482, 502, 503, 508, 486, 15, 11, 6, 0, 0, 0, 0, 0, 529, 476, 538, 520, 519, 552, 492, 518, 0, 0, 0, 0, 0, 0, 0, 0, 494, 524, 501, 500, 518, 485, 540, 509, 33, 26, 0, 0, 0, 0, 0, 0, 456, 526, 504, 501, 478, 486, 508, 549, 22, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4059, 4078, 4135, 4053, 4144, 4130, 4037] avg=4096.0000 max=4144 max_rank=5 min=4037 min_rank=7 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 3968, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[486, 487, 503, 527, 503, 507, 493, 518, 534, 463, 514, 490, 470, 501, 535, 502, 516, 509, 541, 512, 544, 482, 503, 533, 532, 548, 508, 535, 485, 541, 522, 522, 474, 547, 511, 538, 543, 490, 511, 517, 493, 522, 516, 483, 515, 524, 496, 478, 554, 494, 502, 496, 508, 517, 511, 531, 529, 489, 509, 512, 518, 543, 521, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[546, 519, 503, 513, 511, 492, 546, 524, 504, 558, 533, 498, 511, 513, 505, 489, 526, 505, 498, 517, 486, 493, 511, 540, 500, 500, 523, 504, 529, 520, 527, 545, 481, 491, 483, 510, 497, 520, 575, 498, 498, 505, 517, 470, 511, 501, 547, 493, 503, 530, 560, 508, 560, 486, 504, 509, 510, 508, 502, 488, 493, 530, 511, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[499, 541, 492, 521, 501, 499, 492, 532, 508, 493, 502, 526, 521, 519, 546, 509, 542, 524, 503, 543, 503, 506, 503, 530, 497, 453, 499, 510, 515, 530, 524, 503, 509, 519, 531, 548, 498, 522, 502, 515, 528, 527, 526, 515, 539, 526, 504, 513, 491, 513, 515, 464, 531, 549, 491, 511, 506, 507, 500, 479, 519, 477, 497, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[485, 528, 500, 490, 520, 529, 504, 554, 506, 518, 500, 525, 484, 507, 516, 523, 497, 503, 542, 528, 535, 529, 503, 496, 501, 495, 474, 501, 525, 518, 519, 525, 487, 478, 510, 510, 537, 471, 512, 520, 559, 513, 510, 523, 455, 531, 507, 503, 529, 539, 541, 531, 527, 502, 534, 502, 518, 535, 458, 493, 494, 494, 519, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[541, 475, 487, 521, 495, 512, 538, 523, 550, 507, 510, 503, 537, 521, 514, 490, 532, 515, 495, 521, 522, 527, 495, 506, 522, 520, 517, 540, 510, 532, 468, 563, 502, 535, 501, 497, 467, 539, 524, 532, 492, 507, 518, 500, 504, 508, 530, 512, 509, 510, 498, 479, 523, 481, 504, 518, 496, 484, 527, 531, 482, 513, 529, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[514, 520, 508, 513, 509, 537, 512, 511, 550, 512, 538, 525, 524, 488, 515, 511, 544, 506, 521, 511, 520, 544, 510, 508, 507, 508, 499, 514, 488, 493, 527, 503, 477, 483, 529, 519, 515, 498, 555, 481, 498, 504, 545, 531, 512, 497, 516, 508, 526, 487, 513, 441, 527, 504, 534, 510, 506, 507, 526, 494, 499, 533, 492, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[514, 516, 532, 498, 526, 540, 533, 513, 526, 545, 504, 551, 484, 536, 519, 493, 500, 513, 496, 531, 525, 480, 504, 507, 539, 512, 507, 484, 516, 488, 501, 509, 497, 510, 527, 517, 528, 543, 468, 522, 507, 535, 498, 507, 498, 510, 522, 490, 539, 512, 517, 539, 500, 484, 503, 540, 483, 521, 516, 507, 514, 495, 510, 467] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[485, 522, 514, 502, 540, 509, 547, 460, 534, 497, 543, 501, 525, 516, 521, 507, 506, 489, 549, 513, 521, 536, 500, 454, 498, 523, 518, 502, 491, 512, 524, 517, 498, 531, 516, 540, 495, 489, 476, 521, 541, 542, 527, 489, 503, 517, 496, 518, 469, 501, 505, 530, 536, 541, 489, 540, 533, 509, 492, 498, 521, 514, 534, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[486, 487, 503, 527, 503, 507, 487, 518, 0, 0, 0, 0, 0, 0, 0, 0, 515, 463, 514, 490, 470, 501, 535, 502, 0, 0, 0, 0, 0, 0, 0, 0, 516, 509, 541, 496, 544, 482, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 532, 548, 508, 535, 485, 541, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 474, 547, 511, 538, 543, 490, 511, 517, 4, 4, 0, 0, 0, 0, 0, 0, 493, 522, 516, 483, 515, 524, 496, 478, 2, 1, 0, 0, 0, 0, 0, 0, 554, 494, 502, 496, 499, 517, 511, 531, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 509, 512, 518, 543, 521, 510, 19, 16, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 3990, 4124, 4192, 4139, 4030, 4104, 4171] avg=4096.0000 max=4192 max_rank=3 min=3990 min_rank=1 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16444 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 519, 503, 513, 511, 492, 538, 524, 0, 0, 0, 0, 0, 0, 0, 0, 486, 558, 533, 498, 511, 513, 505, 489, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 498, 501, 486, 493, 511, 540, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 523, 504, 529, 520, 527, 544, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 483, 510, 497, 520, 575, 498, 5, 5, 0, 0, 0, 0, 0, 0, 498, 505, 517, 470, 511, 501, 547, 493, 3, 1, 0, 0, 0, 0, 0, 0, 503, 530, 560, 508, 550, 486, 504, 509, 0, 0, 0, 0, 0, 0, 0, 0, 510, 508, 502, 488, 493, 530, 511, 480, 18, 16, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4093, 4060, 4147, 4065, 4046, 4150, 4061] avg=4096.0000 max=4150 max_rank=6 min=4046 min_rank=5 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 528, 500, 490, 520, 529, 498, 554, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 500, 525, 484, 507, 516, 523, 0, 0, 0, 0, 0, 0, 0, 0, 497, 503, 542, 511, 535, 529, 503, 496, 0, 0, 0, 0, 0, 0, 0, 0, 501, 495, 474, 501, 525, 518, 519, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 478, 510, 510, 537, 471, 512, 520, 5, 4, 0, 0, 0, 0, 0, 0, 559, 513, 510, 523, 455, 531, 507, 503, 2, 1, 0, 0, 0, 0, 0, 0, 529, 539, 541, 531, 517, 502, 534, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 458, 493, 494, 494, 519, 546, 18, 17, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4061, 4116, 4057, 4034, 4104, 4195, 4097] avg=4096.0000 max=4195 max_rank=6 min=4034 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16430 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[541, 475, 487, 521, 495, 512, 530, 523, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 510, 503, 537, 521, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 532, 515, 495, 505, 522, 527, 495, 506, 0, 0, 0, 0, 0, 0, 0, 0, 522, 520, 517, 540, 510, 532, 468, 562, 0, 0, 0, 0, 0, 0, 0, 0, 502, 535, 501, 497, 467, 539, 524, 532, 5, 5, 0, 0, 0, 0, 0, 0, 492, 507, 518, 500, 504, 508, 530, 512, 3, 1, 0, 0, 0, 0, 0, 0, 509, 510, 498, 479, 513, 481, 504, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 527, 531, 482, 513, 529, 507, 19, 16, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4113, 4097, 4171, 4107, 4075, 4012, 4109] avg=4096.0000 max=4171 max_rank=3 min=4012 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[499, 541, 492, 521, 501, 499, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0, 490, 493, 502, 526, 521, 519, 546, 509, 0, 0, 0, 0, 0, 0, 0, 0, 542, 524, 503, 526, 503, 506, 503, 530, 0, 0, 0, 0, 0, 0, 0, 0, 497, 453, 499, 510, 515, 530, 524, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 519, 531, 548, 498, 522, 502, 515, 5, 4, 0, 0, 0, 0, 0, 0, 528, 527, 526, 515, 539, 526, 504, 513, 2, 1, 0, 0, 0, 0, 0, 0, 491, 513, 515, 464, 521, 549, 491, 511, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 500, 479, 519, 477, 497, 510, 18, 17, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4106, 4137, 4030, 4153, 4181, 4055, 4035] avg=4096.0000 max=4181 max_rank=5 min=4030 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[485, 522, 514, 502, 540, 509, 539, 460, 0, 0, 0, 0, 0, 0, 0, 0, 515, 497, 543, 501, 525, 516, 521, 507, 0, 0, 0, 0, 0, 0, 0, 0, 506, 489, 549, 497, 521, 536, 500, 454, 0, 0, 0, 0, 0, 0, 0, 0, 498, 523, 518, 502, 491, 512, 524, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 531, 516, 540, 495, 489, 476, 521, 5, 5, 0, 0, 0, 0, 0, 0, 541, 542, 527, 489, 503, 517, 496, 518, 3, 1, 0, 0, 0, 0, 0, 0, 469, 501, 505, 530, 526, 541, 489, 540, 0, 0, 0, 0, 0, 0, 0, 0, 533, 509, 492, 498, 521, 514, 534, 481, 19, 16, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4125, 4052, 4084, 4076, 4137, 4101, 4122] avg=4096.0000 max=4137 max_rank=5 min=4052 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 520, 508, 513, 509, 537, 506, 511, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 538, 525, 524, 488, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 544, 506, 521, 495, 520, 544, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 499, 514, 488, 493, 527, 502, 0, 0, 0, 0, 0, 0, 0, 0, 477, 483, 529, 519, 515, 498, 555, 481, 5, 4, 0, 0, 0, 0, 0, 0, 498, 504, 545, 531, 512, 497, 516, 508, 2, 1, 0, 0, 0, 0, 0, 0, 526, 487, 513, 441, 517, 504, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 506, 507, 526, 494, 499, 533, 492, 511, 19, 16, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4144, 4148, 4038, 4066, 4114, 4032, 4108] avg=4096.0000 max=4148 max_rank=2 min=4032 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[514, 516, 532, 498, 526, 540, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 504, 545, 504, 551, 484, 536, 519, 493, 0, 0, 0, 0, 0, 0, 0, 0, 500, 513, 496, 509, 525, 480, 504, 507, 0, 0, 0, 0, 0, 0, 0, 0, 539, 512, 507, 484, 516, 488, 501, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 510, 527, 517, 528, 543, 468, 522, 7, 9, 0, 0, 0, 0, 0, 0, 507, 535, 498, 507, 498, 510, 522, 490, 7, 7, 0, 0, 0, 0, 0, 0, 539, 512, 517, 539, 485, 484, 503, 540, 0, 0, 0, 0, 0, 0, 0, 0, 483, 521, 516, 507, 514, 495, 510, 467, 22, 22, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4156, 4136, 4034, 4049, 4128, 4081, 4119, 4065] avg=4096.0000 max=4156 max_rank=0 min=4034 min_rank=2 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[487, 533, 530, 519, 524, 470, 531, 534, 498, 472, 559, 529, 507, 482, 527, 494, 524, 518, 534, 527, 496, 516, 529, 505, 507, 498, 524, 527, 488, 494, 489, 495, 507, 524, 525, 505, 520, 505, 516, 504, 564, 508, 474, 499, 528, 532, 537, 532, 503, 491, 512, 524, 510, 500, 476, 532, 467, 536, 478, 524, 503, 513, 504, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[526, 531, 486, 590, 517, 512, 469, 487, 445, 489, 518, 532, 503, 498, 501, 517, 541, 518, 550, 500, 506, 515, 511, 496, 527, 478, 534, 508, 522, 514, 513, 508, 481, 557, 517, 510, 528, 508, 511, 520, 512, 520, 516, 511, 494, 512, 497, 491, 521, 529, 518, 553, 511, 513, 464, 543, 493, 477, 543, 488, 497, 509, 545, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[501, 525, 533, 507, 532, 497, 496, 518, 506, 510, 518, 536, 499, 526, 498, 513, 475, 515, 515, 523, 512, 529, 514, 512, 509, 497, 555, 525, 567, 544, 507, 494, 474, 478, 528, 518, 513, 505, 555, 469, 536, 543, 513, 558, 484, 473, 472, 490, 491, 543, 519, 547, 501, 510, 482, 501, 476, 507, 524, 547, 478, 479, 525, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[531, 523, 526, 509, 557, 499, 494, 561, 486, 525, 523, 555, 529, 503, 515, 511, 525, 498, 474, 483, 483, 488, 505, 505, 493, 544, 516, 516, 531, 491, 529, 510, 501, 506, 493, 504, 491, 501, 496, 532, 503, 492, 545, 516, 532, 533, 526, 538, 514, 506, 517, 496, 485, 481, 518, 533, 533, 502, 533, 509, 507, 495, 517, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[511, 529, 530, 501, 526, 518, 541, 525, 504, 531, 510, 504, 497, 506, 540, 485, 542, 525, 480, 512, 503, 507, 517, 508, 510, 519, 500, 525, 512, 499, 470, 528, 524, 500, 522, 530, 474, 533, 475, 533, 481, 511, 521, 453, 499, 525, 530, 519, 534, 493, 531, 483, 515, 491, 532, 528, 516, 509, 533, 498, 512, 528, 503, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[486, 527, 510, 533, 510, 515, 526, 507, 495, 514, 553, 537, 476, 501, 530, 527, 535, 491, 526, 494, 528, 476, 528, 498, 493, 539, 507, 486, 473, 510, 507, 538, 485, 531, 518, 547, 545, 496, 498, 551, 518, 532, 526, 536, 511, 510, 503, 496, 515, 490, 490, 485, 513, 525, 492, 514, 520, 486, 507, 505, 522, 532, 508, 485] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[529, 510, 493, 523, 512, 520, 529, 489, 503, 520, 499, 492, 508, 530, 509, 532, 518, 512, 522, 488, 502, 514, 514, 518, 486, 512, 523, 522, 527, 526, 543, 493, 511, 499, 508, 516, 553, 469, 500, 530, 469, 526, 502, 482, 512, 521, 530, 529, 475, 564, 539, 491, 506, 513, 518, 519, 525, 492, 494, 486, 525, 509, 502, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[532, 550, 517, 471, 518, 502, 514, 500, 511, 490, 534, 525, 504, 498, 499, 524, 496, 513, 492, 496, 518, 561, 501, 492, 527, 515, 512, 509, 489, 522, 527, 514, 466, 522, 538, 557, 517, 558, 474, 467, 539, 510, 542, 515, 514, 482, 525, 510, 496, 510, 529, 566, 495, 527, 513, 488, 493, 509, 478, 503, 527, 500, 538, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[487, 498, 530, 519, 524, 470, 531, 534, 0, 0, 0, 0, 0, 0, 0, 0, 498, 472, 559, 529, 507, 482, 527, 494, 6, 0, 0, 0, 0, 0, 0, 0, 524, 518, 534, 527, 496, 516, 529, 505, 9, 2, 0, 0, 0, 0, 0, 0, 507, 498, 519, 527, 488, 494, 489, 495, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 525, 505, 520, 505, 516, 504, 2, 0, 0, 0, 0, 0, 0, 0, 564, 499, 474, 499, 528, 532, 537, 532, 0, 0, 0, 0, 0, 0, 0, 0, 503, 491, 512, 524, 510, 500, 476, 532, 3, 1, 0, 0, 0, 0, 0, 0, 467, 536, 478, 524, 503, 513, 504, 548, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4074, 4160, 4017, 4108, 4165, 4052, 4099] avg=4096.0000 max=4165 max_rank=5 min=4017 min_rank=3 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[531, 488, 526, 509, 557, 499, 494, 561, 0, 0, 0, 0, 0, 0, 0, 0, 486, 525, 523, 555, 529, 503, 515, 511, 6, 0, 0, 0, 0, 0, 0, 0, 525, 498, 474, 483, 483, 488, 505, 505, 9, 2, 0, 0, 0, 0, 0, 0, 493, 544, 511, 516, 531, 491, 529, 510, 0, 0, 0, 0, 0, 0, 0, 0, 501, 506, 493, 504, 491, 501, 496, 532, 2, 0, 0, 0, 0, 0, 0, 0, 503, 483, 545, 516, 532, 533, 526, 538, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 517, 496, 485, 481, 518, 533, 3, 1, 0, 0, 0, 0, 0, 0, 533, 502, 533, 509, 507, 495, 517, 475, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4153, 3972, 4125, 4026, 4176, 4054, 4097] avg=4096.0000 max=4176 max_rank=5 min=3972 min_rank=2 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[501, 490, 533, 507, 532, 497, 496, 518, 0, 0, 0, 0, 0, 0, 0, 0, 506, 510, 518, 536, 499, 526, 498, 513, 7, 0, 0, 0, 0, 0, 0, 0, 475, 515, 515, 523, 512, 529, 514, 512, 9, 2, 0, 0, 0, 0, 0, 0, 509, 497, 550, 525, 567, 544, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 474, 478, 528, 518, 513, 505, 555, 469, 2, 0, 0, 0, 0, 0, 0, 0, 536, 533, 513, 558, 484, 473, 472, 490, 0, 0, 0, 0, 0, 0, 0, 0, 491, 543, 519, 547, 501, 510, 482, 501, 3, 1, 0, 0, 0, 0, 0, 0, 476, 507, 524, 547, 478, 479, 525, 521, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4113, 4106, 4193, 4042, 4059, 4098, 4083] avg=4096.0000 max=4193 max_rank=3 min=4042 min_rank=4 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16282 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 496, 486, 590, 517, 512, 469, 487, 0, 0, 0, 0, 0, 0, 0, 0, 445, 489, 518, 532, 503, 498, 501, 517, 7, 0, 0, 0, 0, 0, 0, 0, 541, 518, 550, 500, 506, 515, 511, 496, 9, 2, 0, 0, 0, 0, 0, 0, 527, 478, 529, 508, 522, 514, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 481, 557, 517, 510, 528, 508, 511, 520, 2, 0, 0, 0, 0, 0, 0, 0, 512, 510, 516, 511, 494, 512, 497, 491, 0, 0, 0, 0, 0, 0, 0, 0, 521, 529, 518, 553, 511, 513, 464, 543, 3, 1, 0, 0, 0, 0, 0, 0, 493, 477, 543, 488, 497, 509, 545, 517, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4010, 4148, 4099, 4134, 4043, 4156, 4095] avg=4096.0000 max=4156 max_rank=6 min=4010 min_rank=1 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 476, 493, 523, 512, 520, 529, 489, 0, 0, 0, 0, 0, 0, 0, 0, 503, 520, 499, 492, 508, 530, 509, 532, 7, 0, 0, 0, 0, 0, 0, 0, 518, 512, 522, 488, 502, 514, 514, 518, 9, 2, 0, 0, 0, 0, 0, 0, 486, 512, 518, 522, 527, 526, 543, 493, 0, 0, 0, 0, 0, 0, 0, 0, 511, 499, 508, 516, 553, 469, 500, 530, 2, 0, 0, 0, 0, 0, 0, 0, 469, 516, 502, 482, 512, 521, 530, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 564, 539, 491, 506, 513, 518, 519, 3, 1, 0, 0, 0, 0, 0, 0, 525, 492, 494, 486, 525, 509, 502, 535, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4071, 4100, 4099, 4127, 4088, 4061, 4129, 4093] avg=4096.0000 max=4129 max_rank=6 min=4061 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[532, 513, 517, 471, 518, 502, 514, 500, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 534, 525, 504, 498, 499, 524, 6, 0, 0, 0, 0, 0, 0, 0, 496, 513, 492, 496, 518, 561, 501, 492, 10, 2, 0, 0, 0, 0, 0, 0, 527, 515, 507, 509, 489, 522, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 466, 522, 538, 557, 517, 558, 474, 467, 2, 0, 0, 0, 0, 0, 0, 0, 539, 501, 542, 515, 514, 482, 525, 510, 0, 0, 0, 0, 0, 0, 0, 0, 496, 510, 529, 566, 495, 527, 513, 488, 3, 1, 0, 0, 0, 0, 0, 0, 493, 509, 478, 503, 527, 500, 538, 487, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4091, 4081, 4110, 4101, 4128, 4128, 4062] avg=4096.0000 max=4128 max_rank=5 min=4062 min_rank=7 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 494, 530, 501, 526, 518, 541, 525, 0, 0, 0, 0, 0, 0, 0, 0, 504, 531, 510, 504, 497, 506, 540, 485, 6, 0, 0, 0, 0, 0, 0, 0, 542, 525, 480, 512, 503, 507, 517, 508, 9, 2, 0, 0, 0, 0, 0, 0, 510, 519, 495, 525, 512, 499, 470, 528, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 522, 530, 474, 533, 475, 533, 2, 0, 0, 0, 0, 0, 0, 0, 481, 502, 521, 453, 499, 525, 530, 519, 0, 0, 0, 0, 0, 0, 0, 0, 534, 493, 531, 483, 515, 491, 532, 528, 3, 1, 0, 0, 0, 0, 0, 0, 516, 509, 533, 498, 512, 528, 503, 517, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4083, 4105, 4058, 4093, 4030, 4111, 4142] avg=4096.0000 max=4146 max_rank=0 min=4030 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 482, 510, 533, 510, 515, 526, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 514, 553, 537, 476, 501, 530, 527, 11, 0, 0, 0, 0, 0, 0, 0, 535, 491, 526, 494, 528, 476, 528, 498, 13, 8, 0, 0, 0, 0, 0, 0, 493, 539, 493, 486, 473, 510, 507, 538, 0, 0, 0, 0, 0, 0, 0, 0, 485, 531, 518, 547, 545, 496, 498, 551, 5, 0, 0, 0, 0, 0, 0, 0, 518, 506, 526, 536, 511, 510, 503, 496, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 490, 485, 513, 525, 492, 514, 9, 7, 0, 0, 0, 0, 0, 0, 520, 486, 507, 505, 522, 532, 508, 485, 32, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4069, 4144, 4097, 4039, 4176, 4106, 4040, 4097] avg=4096.0000 max=4176 max_rank=4 min=4039 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4224, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[3968, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 14:57:43] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 277.0 | throughput per GPU (TFLOP/s/GPU): 445.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.224381E+00 | loss scale: 1.0 | grad norm: 5.026 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[497, 523, 524, 532, 503, 528, 512, 511, 510, 545, 532, 538, 491, 522, 525, 523, 531, 538, 501, 494, 488, 549, 489, 507, 531, 516, 485, 538, 543, 550, 519, 499, 495, 527, 495, 494, 495, 499, 447, 544, 520, 503, 498, 517, 487, 469, 530, 495, 515, 520, 528, 517, 534, 514, 483, 542, 516, 489, 526, 495, 497, 512, 482, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[538, 516, 457, 470, 532, 511, 461, 497, 489, 524, 552, 516, 480, 520, 519, 502, 518, 502, 500, 510, 509, 502, 487, 506, 482, 523, 522, 528, 525, 546, 489, 456, 518, 556, 510, 515, 528, 520, 532, 513, 544, 459, 538, 532, 542, 491, 492, 520, 507, 511, 514, 523, 510, 532, 558, 503, 514, 528, 536, 476, 506, 524, 513, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[520, 527, 508, 518, 516, 525, 502, 538, 507, 520, 510, 487, 569, 487, 475, 495, 517, 485, 509, 520, 491, 527, 506, 518, 548, 475, 542, 475, 494, 485, 554, 528, 526, 505, 495, 519, 505, 519, 515, 511, 548, 477, 527, 467, 487, 525, 505, 525, 562, 506, 503, 493, 508, 499, 526, 532, 517, 502, 511, 529, 523, 511, 517, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[520, 486, 526, 545, 552, 521, 496, 503, 536, 499, 519, 519, 480, 483, 503, 510, 487, 509, 514, 549, 506, 505, 491, 496, 483, 534, 507, 507, 528, 532, 525, 507, 495, 499, 503, 539, 515, 530, 521, 552, 498, 511, 535, 549, 500, 509, 511, 518, 511, 494, 522, 468, 501, 523, 482, 512, 527, 466, 495, 511, 514, 505, 517, 557] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[499, 518, 522, 490, 537, 526, 498, 509, 538, 496, 513, 533, 536, 513, 512, 498, 520, 559, 508, 526, 532, 539, 487, 508, 513, 493, 532, 493, 503, 510, 511, 499, 516, 486, 489, 528, 492, 546, 505, 489, 523, 522, 515, 513, 517, 534, 500, 523, 516, 481, 516, 490, 487, 531, 501, 507, 536, 477, 526, 526, 521, 496, 479, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[512, 525, 503, 494, 491, 543, 511, 529, 478, 499, 508, 496, 495, 535, 500, 545, 496, 508, 535, 524, 552, 497, 493, 450, 528, 528, 530, 534, 500, 494, 543, 515, 504, 509, 493, 539, 515, 519, 522, 540, 505, 526, 526, 515, 526, 484, 502, 486, 519, 500, 525, 504, 498, 500, 486, 552, 524, 502, 536, 499, 515, 480, 510, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[540, 499, 470, 472, 493, 484, 508, 552, 466, 516, 494, 540, 489, 488, 530, 485, 518, 502, 523, 535, 482, 497, 515, 501, 526, 520, 529, 547, 532, 521, 538, 529, 489, 519, 496, 493, 506, 538, 521, 551, 510, 486, 487, 523, 524, 502, 488, 483, 512, 532, 485, 562, 522, 515, 539, 516, 493, 562, 525, 529, 496, 467, 539, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[545, 509, 513, 526, 518, 523, 516, 480, 495, 481, 523, 504, 535, 518, 529, 501, 517, 562, 525, 513, 542, 547, 473, 506, 539, 469, 481, 492, 483, 492, 507, 513, 497, 534, 495, 530, 477, 524, 531, 528, 526, 518, 518, 532, 523, 528, 477, 479, 536, 514, 536, 490, 481, 506, 494, 508, 530, 515, 478, 541, 548, 525, 472, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[497, 523, 524, 532, 503, 528, 512, 511, 0, 0, 0, 0, 0, 0, 0, 0, 510, 545, 532, 538, 491, 522, 525, 523, 7, 4, 0, 0, 0, 0, 0, 0, 531, 538, 501, 494, 488, 549, 489, 507, 1, 0, 0, 0, 0, 0, 0, 0, 531, 516, 485, 538, 543, 550, 501, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 495, 494, 495, 499, 447, 533, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 498, 517, 487, 469, 530, 495, 14, 0, 0, 0, 0, 0, 0, 0, 510, 520, 528, 517, 534, 514, 483, 542, 0, 0, 0, 0, 0, 0, 0, 0, 516, 489, 526, 495, 497, 512, 482, 489, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4197, 4098, 4163, 3985, 4033, 4148, 4014] avg=4096.0000 max=4197 max_rank=1 min=3985 min_rank=4 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[538, 516, 457, 470, 532, 511, 461, 497, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 552, 516, 480, 520, 519, 502, 7, 3, 0, 0, 0, 0, 0, 0, 518, 502, 500, 510, 509, 502, 487, 506, 1, 0, 0, 0, 0, 0, 0, 0, 482, 523, 522, 528, 525, 546, 473, 456, 0, 0, 0, 0, 0, 0, 0, 0, 518, 556, 510, 515, 528, 520, 532, 503, 0, 0, 0, 0, 0, 0, 0, 0, 544, 459, 538, 532, 542, 491, 492, 520, 13, 0, 0, 0, 0, 0, 0, 0, 502, 511, 514, 523, 510, 532, 558, 503, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 536, 476, 506, 524, 513, 514, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3982, 4112, 4035, 4055, 4182, 4131, 4153, 4118] avg=4096.0000 max=4182 max_rank=4 min=3982 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 527, 508, 518, 516, 525, 502, 538, 0, 0, 0, 0, 0, 0, 0, 0, 507, 520, 510, 487, 569, 487, 475, 495, 7, 4, 0, 0, 0, 0, 0, 0, 517, 485, 509, 520, 491, 527, 506, 518, 1, 0, 0, 0, 0, 0, 0, 0, 548, 475, 542, 475, 494, 485, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 495, 519, 505, 519, 515, 501, 0, 0, 0, 0, 0, 0, 0, 0, 548, 477, 527, 467, 487, 525, 505, 525, 15, 0, 0, 0, 0, 0, 0, 0, 557, 506, 503, 493, 508, 499, 526, 532, 0, 0, 0, 0, 0, 0, 0, 0, 517, 502, 511, 529, 523, 511, 517, 495, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4061, 4074, 4082, 4085, 4076, 4124, 4112] avg=4096.0000 max=4154 max_rank=0 min=4061 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 486, 526, 545, 552, 521, 496, 503, 0, 0, 0, 0, 0, 0, 0, 0, 536, 499, 519, 519, 480, 483, 503, 510, 7, 4, 0, 0, 0, 0, 0, 0, 487, 509, 514, 549, 506, 505, 491, 496, 1, 0, 0, 0, 0, 0, 0, 0, 483, 534, 507, 507, 528, 532, 507, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 499, 503, 539, 515, 530, 521, 541, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 535, 549, 500, 509, 511, 518, 14, 0, 0, 0, 0, 0, 0, 0, 506, 494, 522, 468, 501, 523, 482, 512, 0, 0, 0, 0, 0, 0, 0, 0, 527, 466, 495, 511, 514, 505, 517, 557, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4060, 4058, 4105, 4143, 4145, 4008, 4100] avg=4096.0000 max=4149 max_rank=0 min=4008 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[540, 499, 470, 472, 493, 484, 508, 552, 0, 0, 0, 0, 0, 0, 0, 0, 466, 516, 494, 540, 489, 488, 530, 485, 7, 4, 0, 0, 0, 0, 0, 0, 518, 502, 523, 535, 482, 497, 515, 501, 1, 0, 0, 0, 0, 0, 0, 0, 526, 520, 529, 547, 532, 521, 519, 529, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 496, 493, 506, 538, 521, 540, 0, 0, 0, 0, 0, 0, 0, 0, 510, 486, 487, 523, 524, 502, 488, 483, 15, 0, 0, 0, 0, 0, 0, 0, 507, 532, 485, 562, 522, 515, 539, 516, 0, 0, 0, 0, 0, 0, 0, 0, 493, 562, 525, 529, 496, 467, 539, 517, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4018, 4019, 4074, 4223, 4102, 4018, 4178, 4136] avg=4096.0000 max=4223 max_rank=3 min=4018 min_rank=0 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[499, 518, 522, 490, 537, 526, 498, 509, 0, 0, 0, 0, 0, 0, 0, 0, 538, 496, 513, 533, 536, 513, 512, 498, 6, 4, 0, 0, 0, 0, 0, 0, 520, 559, 508, 526, 532, 539, 487, 508, 1, 0, 0, 0, 0, 0, 0, 0, 513, 493, 532, 493, 503, 510, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0, 516, 486, 489, 528, 492, 546, 505, 480, 0, 0, 0, 0, 0, 0, 0, 0, 523, 522, 515, 513, 517, 534, 500, 523, 14, 0, 0, 0, 0, 0, 0, 0, 511, 481, 516, 490, 487, 531, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 536, 477, 526, 526, 521, 496, 479, 509, 4, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4149, 4180, 4036, 4042, 4161, 4024, 4077] avg=4096.0000 max=4180 max_rank=2 min=4024 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[512, 525, 503, 494, 491, 543, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 478, 499, 508, 496, 495, 535, 500, 545, 7, 4, 0, 0, 0, 0, 0, 0, 496, 508, 535, 524, 552, 497, 493, 450, 1, 0, 0, 0, 0, 0, 0, 0, 528, 528, 530, 534, 500, 494, 524, 515, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 493, 539, 515, 519, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 526, 526, 515, 526, 484, 502, 486, 15, 0, 0, 0, 0, 0, 0, 0, 514, 500, 525, 504, 498, 500, 486, 552, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 536, 499, 515, 480, 510, 516, 4, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4067, 4056, 4153, 4130, 4085, 4079, 4090] avg=4096.0000 max=4153 max_rank=3 min=4056 min_rank=2 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[543, 509, 513, 526, 518, 523, 516, 480, 0, 0, 0, 0, 0, 0, 0, 0, 495, 481, 523, 504, 535, 518, 529, 501, 10, 7, 0, 0, 0, 0, 0, 0, 517, 562, 525, 513, 542, 547, 473, 506, 6, 2, 0, 0, 0, 0, 0, 0, 539, 469, 481, 492, 483, 492, 482, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 534, 495, 530, 477, 524, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 526, 518, 518, 532, 523, 528, 477, 479, 18, 0, 0, 0, 0, 0, 0, 0, 525, 514, 536, 490, 481, 506, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 530, 515, 478, 541, 548, 525, 472, 500, 5, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4103, 4193, 3951, 4099, 4119, 4054, 4121] avg=4096.0000 max=4193 max_rank=2 min=3951 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[535, 486, 500, 513, 544, 493, 537, 504, 483, 498, 554, 494, 519, 541, 497, 499, 515, 528, 483, 499, 515, 540, 512, 509, 506, 514, 549, 517, 503, 484, 502, 515, 506, 504, 515, 551, 538, 503, 512, 514, 519, 503, 513, 516, 508, 483, 538, 508, 511, 483, 522, 514, 560, 511, 531, 537, 472, 466, 503, 490, 506, 519, 508, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[541, 482, 515, 511, 488, 511, 530, 498, 510, 527, 531, 519, 492, 511, 511, 532, 489, 524, 492, 509, 522, 532, 534, 528, 541, 535, 520, 501, 492, 551, 474, 494, 509, 530, 506, 516, 517, 516, 442, 551, 523, 485, 484, 503, 488, 479, 502, 555, 518, 514, 518, 508, 526, 510, 517, 498, 494, 545, 498, 502, 530, 503, 483, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[476, 522, 526, 527, 534, 513, 519, 524, 497, 532, 484, 501, 522, 507, 497, 515, 518, 524, 520, 547, 491, 488, 485, 537, 515, 511, 503, 479, 519, 516, 492, 529, 517, 510, 511, 502, 496, 534, 558, 484, 483, 512, 512, 487, 496, 548, 512, 509, 504, 507, 499, 522, 525, 496, 495, 509, 517, 507, 535, 497, 525, 508, 527, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[526, 516, 529, 497, 479, 515, 478, 487, 527, 520, 520, 489, 502, 535, 516, 529, 500, 516, 526, 536, 565, 491, 536, 515, 517, 545, 510, 552, 499, 522, 505, 518, 506, 491, 502, 499, 530, 527, 490, 521, 516, 473, 534, 515, 502, 472, 490, 502, 520, 551, 501, 543, 510, 511, 500, 509, 504, 523, 509, 488, 508, 512, 490, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[514, 508, 509, 507, 568, 522, 560, 529, 538, 496, 483, 536, 535, 508, 510, 551, 487, 536, 462, 515, 505, 532, 491, 538, 532, 525, 458, 508, 523, 465, 512, 525, 506, 488, 507, 515, 527, 506, 513, 491, 514, 509, 525, 506, 518, 502, 543, 532, 467, 498, 527, 516, 504, 489, 510, 503, 489, 515, 509, 502, 539, 521, 513, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[520, 517, 471, 539, 526, 530, 505, 490, 492, 519, 482, 496, 521, 492, 518, 518, 502, 536, 523, 515, 508, 515, 512, 490, 508, 535, 496, 504, 530, 502, 471, 521, 480, 545, 536, 511, 506, 503, 576, 577, 517, 530, 503, 541, 534, 471, 528, 495, 536, 471, 509, 472, 496, 507, 523, 520, 538, 531, 534, 486, 522, 477, 521, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[517, 521, 496, 558, 514, 488, 502, 528, 515, 510, 514, 485, 547, 528, 503, 502, 484, 529, 540, 500, 558, 478, 539, 479, 512, 507, 483, 523, 526, 526, 522, 507, 508, 543, 471, 489, 528, 540, 550, 486, 528, 468, 500, 514, 511, 545, 498, 495, 530, 468, 505, 511, 525, 549, 555, 513, 475, 515, 510, 512, 500, 495, 508, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[499, 507, 544, 502, 528, 479, 539, 505, 484, 470, 546, 514, 521, 504, 491, 515, 522, 554, 525, 497, 540, 507, 476, 538, 497, 464, 504, 513, 501, 538, 515, 523, 519, 547, 500, 530, 485, 502, 487, 500, 489, 507, 518, 506, 506, 516, 513, 541, 531, 510, 519, 509, 528, 490, 502, 548, 530, 538, 516, 485, 504, 538, 505, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[541, 482, 515, 511, 474, 511, 530, 498, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 531, 519, 492, 511, 511, 532, 1, 0, 0, 0, 0, 0, 0, 0, 489, 501, 492, 509, 522, 532, 534, 528, 0, 0, 0, 0, 0, 0, 0, 0, 541, 535, 520, 501, 492, 551, 474, 494, 5, 2, 0, 0, 0, 0, 0, 0, 509, 517, 506, 516, 517, 516, 442, 551, 0, 0, 0, 0, 0, 0, 0, 0, 523, 485, 484, 503, 488, 479, 502, 555, 11, 9, 0, 0, 0, 0, 0, 0, 518, 514, 518, 508, 520, 510, 517, 498, 0, 0, 0, 0, 0, 0, 0, 0, 494, 545, 498, 502, 530, 503, 483, 551, 23, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4134, 4107, 4115, 4074, 4039, 4103, 4134] avg=4096.0000 max=4134 max_rank=1 min=4039 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[535, 486, 500, 513, 528, 493, 537, 504, 0, 0, 0, 0, 0, 0, 0, 0, 483, 498, 554, 494, 519, 541, 497, 499, 1, 0, 0, 0, 0, 0, 0, 0, 515, 505, 483, 499, 515, 540, 512, 509, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 549, 517, 503, 484, 502, 515, 5, 2, 0, 0, 0, 0, 0, 0, 506, 492, 515, 551, 538, 503, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 519, 503, 513, 516, 508, 483, 538, 508, 10, 11, 0, 0, 0, 0, 0, 0, 511, 483, 522, 514, 554, 511, 531, 537, 0, 0, 0, 0, 0, 0, 0, 0, 472, 466, 503, 490, 506, 519, 508, 516, 23, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4086, 4078, 4097, 4131, 4109, 4163, 4008] avg=4096.0000 max=4163 max_rank=6 min=4008 min_rank=7 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16411 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 508, 509, 507, 551, 522, 560, 529, 0, 0, 0, 0, 0, 0, 0, 0, 538, 496, 483, 536, 535, 508, 510, 551, 1, 0, 0, 0, 0, 0, 0, 0, 487, 512, 462, 515, 505, 532, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 532, 525, 458, 508, 523, 465, 512, 525, 4, 2, 0, 0, 0, 0, 0, 0, 506, 476, 507, 515, 527, 506, 513, 491, 0, 0, 0, 0, 0, 0, 0, 0, 514, 509, 525, 506, 518, 502, 543, 532, 10, 11, 0, 0, 0, 0, 0, 0, 467, 498, 527, 516, 499, 489, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 489, 515, 509, 502, 539, 521, 513, 476, 24, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4200, 4158, 4042, 4054, 4041, 4170, 4009, 4094] avg=4096.0000 max=4200 max_rank=0 min=4009 min_rank=6 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16314 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[476, 522, 526, 527, 519, 513, 519, 524, 0, 0, 0, 0, 0, 0, 0, 0, 497, 532, 484, 501, 522, 507, 497, 515, 1, 0, 0, 0, 0, 0, 0, 0, 518, 501, 520, 547, 491, 488, 485, 537, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 503, 479, 519, 516, 492, 529, 5, 2, 0, 0, 0, 0, 0, 0, 517, 497, 511, 502, 496, 534, 558, 484, 0, 0, 0, 0, 0, 0, 0, 0, 483, 512, 512, 487, 496, 548, 512, 509, 11, 10, 0, 0, 0, 0, 0, 0, 504, 507, 499, 522, 519, 496, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 517, 507, 535, 497, 525, 508, 527, 554, 23, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4056, 4087, 4071, 4099, 4080, 4051, 4198] avg=4096.0000 max=4198 max_rank=7 min=4051 min_rank=6 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[520, 517, 471, 539, 511, 530, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0, 492, 519, 482, 496, 521, 492, 518, 518, 1, 0, 0, 0, 0, 0, 0, 0, 502, 512, 523, 515, 508, 515, 512, 490, 0, 0, 0, 0, 0, 0, 0, 0, 508, 535, 496, 504, 530, 502, 471, 521, 4, 2, 0, 0, 0, 0, 0, 0, 480, 532, 536, 511, 506, 503, 576, 577, 0, 0, 0, 0, 0, 0, 0, 0, 517, 530, 503, 541, 534, 471, 528, 495, 11, 10, 0, 0, 0, 0, 0, 0, 536, 471, 509, 472, 491, 507, 523, 520, 0, 0, 0, 0, 0, 0, 0, 0, 538, 531, 534, 486, 522, 477, 521, 468, 24, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4039, 4077, 4073, 4221, 4140, 4029, 4106] avg=4096.0000 max=4221 max_rank=4 min=4029 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16496 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[517, 521, 496, 558, 499, 488, 502, 528, 0, 0, 0, 0, 0, 0, 0, 0, 515, 510, 514, 485, 547, 528, 503, 502, 1, 0, 0, 0, 0, 0, 0, 0, 484, 506, 540, 500, 558, 478, 539, 479, 0, 0, 0, 0, 0, 0, 0, 0, 512, 507, 483, 523, 526, 526, 522, 507, 5, 2, 0, 0, 0, 0, 0, 0, 508, 530, 471, 489, 528, 540, 550, 486, 0, 0, 0, 0, 0, 0, 0, 0, 528, 468, 500, 514, 511, 545, 498, 495, 11, 10, 0, 0, 0, 0, 0, 0, 530, 468, 505, 511, 519, 549, 555, 513, 0, 0, 0, 0, 0, 0, 0, 0, 475, 515, 510, 512, 500, 495, 508, 482, 23, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4105, 4084, 4113, 4102, 4080, 4150, 4025] avg=4096.0000 max=4150 max_rank=6 min=4025 min_rank=7 imbalance_ratio=1.013184 max_load_reduction=28618 max_load_reduction_pct=0.873352 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 516, 529, 497, 456, 515, 478, 487, 0, 0, 0, 0, 0, 0, 0, 0, 527, 520, 520, 489, 502, 535, 516, 529, 6, 0, 0, 0, 0, 0, 0, 0, 500, 489, 526, 536, 565, 491, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 517, 545, 510, 552, 499, 522, 505, 518, 7, 8, 0, 0, 0, 0, 0, 0, 506, 468, 502, 499, 530, 527, 490, 521, 0, 0, 0, 0, 0, 0, 0, 0, 516, 473, 534, 515, 502, 472, 490, 502, 15, 14, 0, 0, 0, 0, 0, 0, 520, 551, 501, 543, 497, 511, 500, 509, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 509, 488, 508, 512, 490, 501, 27, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4004, 4144, 4158, 4183, 4043, 4033, 4132, 4071] avg=4096.0000 max=4183 max_rank=3 min=4004 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[499, 507, 544, 502, 513, 479, 539, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 470, 546, 514, 521, 504, 491, 515, 1, 0, 0, 0, 0, 0, 0, 0, 522, 530, 525, 497, 540, 507, 476, 538, 0, 0, 0, 0, 0, 0, 0, 0, 497, 464, 504, 513, 501, 538, 515, 523, 5, 2, 0, 0, 0, 0, 0, 0, 519, 534, 500, 530, 485, 502, 487, 500, 0, 0, 0, 0, 0, 0, 0, 0, 489, 507, 518, 506, 506, 516, 513, 541, 11, 10, 0, 0, 0, 0, 0, 0, 531, 510, 519, 509, 522, 490, 502, 548, 0, 0, 0, 0, 0, 0, 0, 0, 530, 538, 516, 485, 504, 538, 505, 487, 24, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4046, 4135, 4062, 4057, 4117, 4131, 4132] avg=4096.0000 max=4135 max_rank=2 min=4046 min_rank=1 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[489, 524, 521, 534, 518, 531, 488, 516, 506, 493, 524, 472, 545, 502, 503, 528, 529, 525, 498, 518, 519, 511, 534, 472, 494, 479, 482, 512, 548, 550, 512, 541, 481, 491, 513, 498, 523, 522, 531, 501, 520, 531, 474, 504, 488, 525, 491, 528, 489, 506, 508, 522, 507, 517, 562, 515, 456, 537, 527, 511, 504, 536, 525, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[564, 518, 499, 500, 538, 508, 509, 522, 513, 492, 543, 511, 524, 526, 525, 494, 503, 514, 510, 523, 522, 474, 558, 490, 501, 515, 507, 505, 530, 474, 510, 507, 493, 503, 514, 460, 539, 523, 501, 505, 537, 511, 530, 526, 490, 512, 497, 512, 539, 486, 491, 534, 515, 477, 523, 506, 479, 504, 518, 524, 470, 548, 524, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[521, 537, 503, 504, 522, 508, 542, 505, 508, 513, 502, 493, 513, 526, 494, 520, 491, 526, 503, 506, 540, 499, 527, 537, 516, 497, 501, 547, 500, 471, 545, 516, 504, 547, 498, 510, 518, 505, 524, 524, 483, 509, 478, 490, 495, 475, 510, 488, 554, 511, 516, 552, 458, 506, 534, 512, 519, 527, 536, 523, 508, 510, 506, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[545, 477, 518, 488, 531, 507, 525, 536, 502, 514, 527, 523, 515, 538, 462, 498, 500, 498, 534, 526, 489, 525, 488, 527, 536, 462, 501, 577, 538, 535, 526, 506, 555, 495, 542, 502, 491, 496, 523, 500, 493, 475, 531, 497, 528, 482, 511, 499, 518, 507, 494, 501, 496, 513, 504, 490, 538, 510, 529, 476, 524, 548, 543, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[521, 527, 522, 526, 520, 499, 524, 515, 512, 530, 514, 532, 500, 504, 521, 505, 486, 513, 505, 499, 539, 515, 510, 516, 504, 515, 541, 541, 516, 514, 538, 514, 503, 523, 472, 514, 473, 515, 504, 536, 529, 506, 497, 493, 489, 536, 450, 524, 507, 501, 476, 507, 543, 509, 529, 492, 523, 499, 504, 497, 517, 551, 517, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[498, 521, 508, 504, 498, 549, 584, 506, 515, 496, 489, 535, 498, 546, 492, 486, 529, 553, 519, 477, 536, 503, 556, 512, 509, 513, 500, 480, 535, 535, 505, 506, 482, 525, 470, 507, 458, 517, 498, 512, 517, 490, 522, 511, 505, 524, 492, 527, 515, 512, 516, 487, 544, 526, 479, 561, 503, 502, 502, 530, 535, 476, 506, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[557, 525, 533, 513, 524, 500, 545, 468, 510, 527, 540, 500, 533, 539, 499, 522, 477, 531, 528, 509, 512, 563, 524, 492, 459, 537, 529, 530, 502, 493, 504, 531, 526, 529, 501, 525, 562, 507, 469, 476, 528, 505, 489, 473, 493, 493, 480, 543, 494, 520, 525, 507, 496, 488, 494, 529, 529, 533, 470, 527, 470, 491, 551, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[493, 528, 517, 546, 535, 489, 513, 498, 520, 517, 519, 535, 540, 531, 494, 533, 488, 558, 504, 524, 519, 437, 526, 493, 530, 531, 535, 540, 476, 501, 481, 525, 544, 504, 589, 515, 482, 494, 504, 486, 495, 498, 492, 499, 526, 496, 527, 517, 482, 482, 495, 516, 543, 520, 502, 514, 494, 526, 512, 494, 496, 517, 524, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[489, 524, 521, 534, 518, 531, 444, 516, 0, 0, 0, 0, 0, 0, 0, 0, 506, 493, 524, 472, 545, 489, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 529, 525, 498, 518, 519, 511, 522, 472, 0, 0, 0, 0, 0, 0, 0, 0, 494, 479, 482, 496, 548, 550, 512, 541, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 513, 498, 523, 522, 531, 501, 13, 6, 5, 0, 0, 0, 0, 0, 520, 531, 474, 504, 488, 525, 491, 528, 44, 11, 0, 0, 0, 0, 0, 0, 489, 506, 508, 522, 507, 517, 562, 515, 6, 5, 0, 0, 0, 0, 0, 0, 456, 537, 527, 511, 504, 536, 520, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4060, 4094, 4102, 4084, 4116, 4137, 4098] avg=4096.0000 max=4137 max_rank=6 min=4060 min_rank=1 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[564, 518, 499, 500, 538, 508, 463, 522, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 543, 511, 524, 512, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 510, 523, 522, 474, 546, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 515, 507, 489, 530, 474, 510, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 503, 514, 460, 539, 523, 501, 505, 14, 6, 5, 0, 0, 0, 0, 0, 537, 511, 530, 526, 490, 512, 497, 512, 46, 11, 0, 0, 0, 0, 0, 0, 539, 486, 491, 534, 515, 477, 523, 506, 6, 5, 0, 0, 0, 0, 0, 0, 479, 504, 518, 524, 470, 548, 519, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4114, 4082, 4033, 4063, 4172, 4082, 4110] avg=4096.0000 max=4172 max_rank=5 min=4033 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16427 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 537, 503, 504, 522, 508, 493, 505, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 502, 493, 513, 512, 494, 520, 0, 0, 0, 0, 0, 0, 0, 0, 491, 526, 503, 506, 540, 499, 515, 537, 0, 0, 0, 0, 0, 0, 0, 0, 516, 497, 501, 529, 500, 471, 545, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 547, 498, 510, 518, 505, 524, 524, 14, 6, 6, 0, 0, 0, 0, 0, 483, 509, 478, 490, 495, 475, 510, 488, 49, 12, 0, 0, 0, 0, 0, 0, 554, 511, 516, 552, 458, 506, 534, 512, 6, 5, 0, 0, 0, 0, 0, 0, 519, 527, 536, 523, 508, 510, 501, 505, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4055, 4117, 4075, 4156, 3989, 4154, 4129] avg=4096.0000 max=4156 max_rank=4 min=3989 min_rank=5 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 527, 522, 526, 520, 499, 477, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 530, 514, 532, 500, 491, 521, 505, 0, 0, 0, 0, 0, 0, 0, 0, 486, 513, 505, 499, 539, 515, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 541, 523, 516, 514, 538, 514, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 472, 514, 473, 515, 504, 536, 13, 6, 6, 0, 0, 0, 0, 0, 529, 506, 497, 493, 489, 536, 450, 524, 47, 12, 0, 0, 0, 0, 0, 0, 507, 501, 476, 507, 543, 509, 529, 492, 5, 5, 0, 0, 0, 0, 0, 0, 523, 499, 504, 497, 517, 551, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4105, 4072, 4165, 4065, 4083, 4074, 4097] avg=4096.0000 max=4165 max_rank=3 min=4065 min_rank=4 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[545, 477, 518, 488, 531, 507, 478, 536, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 527, 523, 515, 524, 462, 498, 0, 0, 0, 0, 0, 0, 0, 0, 500, 498, 534, 526, 489, 525, 477, 527, 0, 0, 0, 0, 0, 0, 0, 0, 536, 462, 501, 558, 538, 535, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 555, 495, 542, 502, 491, 496, 523, 500, 14, 6, 6, 0, 0, 0, 0, 0, 493, 475, 531, 497, 528, 482, 511, 499, 47, 13, 0, 0, 0, 0, 0, 0, 518, 507, 494, 501, 496, 513, 504, 490, 5, 5, 0, 0, 0, 0, 0, 0, 538, 510, 529, 476, 524, 548, 538, 483, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4065, 4076, 4162, 4130, 4076, 4033, 4146] avg=4096.0000 max=4162 max_rank=3 min=4033 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[493, 528, 517, 546, 535, 489, 467, 498, 0, 0, 0, 0, 0, 0, 0, 0, 520, 517, 519, 535, 540, 517, 494, 533, 0, 0, 0, 0, 0, 0, 0, 0, 488, 558, 504, 524, 519, 437, 514, 493, 0, 0, 0, 0, 0, 0, 0, 0, 530, 531, 535, 523, 476, 501, 481, 525, 0, 0, 0, 0, 0, 0, 0, 0, 544, 504, 589, 515, 482, 494, 504, 486, 14, 6, 5, 0, 0, 0, 0, 0, 495, 498, 492, 499, 526, 496, 527, 517, 46, 12, 0, 0, 0, 0, 0, 0, 482, 482, 495, 516, 543, 520, 502, 514, 6, 5, 0, 0, 0, 0, 0, 0, 494, 526, 512, 494, 496, 517, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4175, 4037, 4102, 4143, 4108, 4065, 4065] avg=4096.0000 max=4175 max_rank=1 min=4037 min_rank=2 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[557, 525, 533, 513, 524, 500, 496, 468, 0, 0, 0, 0, 0, 0, 0, 0, 510, 527, 540, 500, 533, 525, 499, 522, 0, 0, 0, 0, 0, 0, 0, 0, 477, 531, 528, 509, 512, 563, 512, 492, 0, 0, 0, 0, 0, 0, 0, 0, 459, 537, 529, 513, 502, 493, 504, 531, 0, 0, 0, 0, 0, 0, 0, 0, 526, 529, 501, 525, 562, 507, 469, 476, 14, 6, 5, 0, 0, 0, 0, 0, 528, 505, 489, 473, 493, 493, 480, 543, 49, 12, 0, 0, 0, 0, 0, 0, 494, 520, 525, 507, 496, 488, 494, 529, 6, 5, 0, 0, 0, 0, 0, 0, 529, 533, 470, 527, 470, 491, 546, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4156, 4124, 4068, 4120, 4065, 4064, 4055] avg=4096.0000 max=4156 max_rank=1 min=4055 min_rank=7 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16304 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[498, 521, 508, 504, 498, 549, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 496, 489, 535, 498, 527, 492, 486, 0, 0, 0, 0, 0, 0, 0, 0, 529, 553, 519, 477, 536, 503, 537, 512, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 500, 458, 535, 535, 505, 506, 0, 0, 0, 0, 0, 0, 0, 0, 482, 525, 470, 507, 458, 517, 498, 512, 19, 10, 9, 0, 0, 0, 0, 0, 517, 490, 522, 511, 505, 524, 492, 527, 58, 13, 0, 0, 0, 0, 0, 0, 515, 512, 516, 487, 544, 526, 479, 561, 9, 10, 0, 0, 0, 0, 0, 0, 503, 502, 502, 530, 535, 476, 496, 524, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4038, 4166, 4061, 4007, 4159, 4159, 4068] avg=4096.0000 max=4166 max_rank=2 min=4007 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 3968, 4224, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[527, 473, 513, 540, 559, 509, 541, 512, 465, 506, 503, 537, 527, 523, 498, 489, 525, 488, 500, 537, 489, 505, 475, 530, 535, 517, 499, 552, 523, 510, 512, 485, 504, 502, 507, 494, 491, 518, 461, 527, 506, 533, 535, 515, 531, 511, 532, 513, 514, 493, 550, 499, 505, 525, 524, 520, 492, 499, 505, 469, 525, 512, 531, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[527, 516, 502, 501, 480, 527, 502, 514, 489, 481, 498, 514, 541, 517, 510, 517, 520, 521, 554, 505, 519, 551, 554, 526, 513, 514, 514, 512, 545, 507, 517, 503, 508, 520, 524, 549, 497, 543, 506, 498, 516, 534, 491, 517, 484, 482, 505, 528, 468, 565, 511, 507, 483, 511, 508, 510, 491, 503, 526, 466, 463, 481, 543, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 497, 519, 526, 509, 516, 547, 520, 540, 496, 505, 539, 522, 486, 540, 536, 510, 554, 521, 512, 539, 530, 529, 516, 544, 518, 528, 499, 551, 516, 473, 508, 515, 496, 509, 516, 493, 479, 522, 488, 536, 517, 508, 528, 515, 478, 463, 499, 484, 492, 544, 477, 505, 489, 480, 503, 524, 493, 522, 483, 532, 495, 520, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[502, 513, 540, 488, 508, 524, 493, 545, 513, 478, 539, 512, 488, 501, 492, 526, 531, 511, 514, 538, 509, 524, 518, 533, 489, 521, 516, 502, 509, 552, 506, 489, 509, 505, 508, 494, 463, 534, 483, 512, 513, 521, 537, 501, 489, 505, 558, 520, 532, 527, 513, 515, 538, 469, 535, 481, 502, 541, 503, 497, 528, 500, 510, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[538, 523, 512, 500, 495, 526, 512, 474, 486, 511, 514, 551, 501, 523, 492, 531, 501, 547, 512, 521, 516, 542, 528, 540, 531, 526, 541, 496, 557, 520, 488, 490, 512, 532, 512, 524, 505, 492, 490, 494, 528, 524, 500, 495, 491, 529, 521, 514, 490, 508, 513, 485, 522, 514, 507, 506, 509, 495, 481, 517, 516, 486, 503, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[498, 523, 510, 464, 487, 512, 533, 536, 572, 485, 499, 536, 546, 506, 536, 534, 493, 467, 522, 515, 509, 546, 538, 506, 524, 502, 501, 532, 502, 498, 499, 485, 490, 490, 531, 490, 538, 484, 520, 523, 486, 531, 510, 528, 537, 509, 502, 525, 509, 535, 508, 516, 496, 505, 512, 499, 528, 521, 512, 498, 501, 508, 492, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[515, 512, 483, 544, 524, 528, 519, 535, 504, 503, 529, 516, 498, 499, 508, 508, 519, 534, 494, 509, 529, 523, 526, 519, 460, 518, 512, 487, 532, 503, 504, 521, 564, 526, 502, 532, 505, 502, 462, 525, 515, 502, 532, 503, 502, 559, 529, 450, 539, 497, 516, 485, 522, 509, 487, 509, 505, 484, 495, 538, 512, 514, 506, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[547, 535, 541, 533, 502, 498, 502, 524, 527, 508, 488, 464, 500, 504, 520, 512, 544, 530, 511, 492, 496, 485, 525, 521, 521, 503, 519, 492, 510, 511, 477, 503, 513, 516, 515, 532, 546, 518, 509, 518, 481, 516, 498, 499, 520, 550, 554, 566, 509, 500, 507, 513, 504, 503, 500, 473, 531, 484, 497, 489, 504, 512, 545, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 512, 500, 495, 526, 512, 474, 0, 0, 0, 0, 0, 0, 0, 0, 486, 511, 514, 551, 501, 523, 492, 531, 3, 0, 0, 0, 0, 0, 0, 0, 501, 547, 512, 521, 516, 481, 528, 540, 0, 0, 0, 0, 0, 0, 0, 0, 531, 526, 541, 496, 557, 520, 488, 490, 3, 0, 0, 0, 0, 0, 0, 0, 512, 532, 512, 524, 505, 492, 490, 494, 13, 9, 0, 0, 0, 0, 0, 0, 528, 505, 500, 495, 491, 529, 521, 514, 0, 0, 0, 0, 0, 0, 0, 0, 490, 508, 513, 485, 522, 514, 507, 506, 20, 17, 0, 0, 0, 0, 0, 0, 509, 495, 481, 517, 516, 486, 503, 508, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4112, 4146, 4152, 4083, 4083, 4082, 4059] avg=4096.0000 max=4152 max_rank=3 min=4051 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 516, 502, 501, 480, 527, 502, 514, 0, 0, 0, 0, 0, 0, 0, 0, 489, 481, 498, 514, 541, 517, 510, 517, 3, 0, 0, 0, 0, 0, 0, 0, 520, 521, 554, 505, 519, 489, 554, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 514, 514, 512, 545, 507, 517, 503, 3, 0, 0, 0, 0, 0, 0, 0, 508, 520, 524, 549, 497, 543, 506, 498, 13, 9, 0, 0, 0, 0, 0, 0, 516, 515, 491, 517, 484, 482, 505, 528, 0, 0, 0, 0, 0, 0, 0, 0, 468, 565, 511, 507, 483, 511, 508, 510, 19, 17, 0, 0, 0, 0, 0, 0, 491, 503, 526, 466, 463, 481, 543, 519, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4070, 4188, 4128, 4167, 4038, 4099, 4037] avg=4096.0000 max=4188 max_rank=2 min=4037 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16341 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[497, 497, 519, 526, 509, 516, 547, 520, 0, 0, 0, 0, 0, 0, 0, 0, 540, 496, 505, 539, 522, 486, 540, 536, 3, 0, 0, 0, 0, 0, 0, 0, 510, 554, 521, 512, 539, 470, 529, 516, 0, 0, 0, 0, 0, 0, 0, 0, 544, 518, 528, 499, 551, 516, 473, 508, 2, 0, 0, 0, 0, 0, 0, 0, 515, 496, 509, 516, 493, 479, 522, 488, 13, 9, 0, 0, 0, 0, 0, 0, 536, 499, 508, 528, 515, 478, 463, 499, 0, 0, 0, 0, 0, 0, 0, 0, 484, 492, 544, 477, 505, 489, 480, 503, 19, 17, 0, 0, 0, 0, 0, 0, 524, 493, 522, 483, 532, 495, 520, 492, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4167, 4151, 4139, 4040, 4026, 4010, 4104] avg=4096.0000 max=4167 max_rank=1 min=4010 min_rank=6 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16180 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[475, 513, 540, 488, 508, 524, 493, 545, 0, 0, 0, 0, 0, 0, 0, 0, 513, 478, 539, 512, 488, 501, 492, 526, 3, 0, 0, 0, 0, 0, 0, 0, 531, 511, 514, 538, 509, 466, 518, 533, 0, 0, 0, 0, 0, 0, 0, 0, 489, 521, 516, 502, 509, 552, 506, 489, 2, 0, 0, 0, 0, 0, 0, 0, 509, 505, 508, 494, 463, 534, 483, 512, 13, 9, 0, 0, 0, 0, 0, 0, 513, 503, 537, 501, 489, 505, 558, 520, 0, 0, 0, 0, 0, 0, 0, 0, 532, 527, 513, 515, 538, 469, 535, 481, 18, 16, 0, 0, 0, 0, 0, 0, 502, 541, 503, 497, 528, 500, 510, 501, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4052, 4120, 4086, 4030, 4126, 4144, 4124] avg=4096.0000 max=4144 max_rank=6 min=4030 min_rank=4 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 473, 513, 540, 559, 509, 541, 512, 0, 0, 0, 0, 0, 0, 0, 0, 465, 506, 503, 537, 527, 523, 498, 489, 3, 0, 0, 0, 0, 0, 0, 0, 525, 488, 500, 537, 489, 448, 475, 530, 0, 0, 0, 0, 0, 0, 0, 0, 535, 517, 499, 552, 523, 510, 512, 485, 3, 0, 0, 0, 0, 0, 0, 0, 504, 502, 507, 494, 491, 518, 461, 527, 13, 9, 0, 0, 0, 0, 0, 0, 506, 514, 535, 515, 531, 511, 532, 513, 0, 0, 0, 0, 0, 0, 0, 0, 514, 493, 550, 499, 505, 525, 524, 520, 19, 16, 0, 0, 0, 0, 0, 0, 492, 499, 505, 469, 525, 512, 531, 521, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4051, 3992, 4136, 4026, 4157, 4165, 4095] avg=4096.0000 max=4165 max_rank=6 min=3992 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[472, 523, 510, 464, 487, 512, 533, 536, 0, 0, 0, 0, 0, 0, 0, 0, 572, 485, 499, 536, 546, 506, 536, 534, 3, 0, 0, 0, 0, 0, 0, 0, 493, 467, 522, 515, 509, 485, 538, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 502, 501, 532, 502, 498, 499, 485, 3, 0, 0, 0, 0, 0, 0, 0, 490, 490, 531, 490, 538, 484, 520, 523, 13, 8, 0, 0, 0, 0, 0, 0, 486, 512, 510, 528, 537, 509, 502, 525, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 508, 516, 496, 505, 512, 499, 18, 17, 0, 0, 0, 0, 0, 0, 528, 521, 512, 498, 501, 508, 492, 518, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4217, 4035, 4046, 4087, 4109, 4115, 4122] avg=4096.0000 max=4217 max_rank=1 min=4035 min_rank=2 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[478, 512, 483, 544, 524, 528, 519, 535, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 529, 516, 498, 499, 508, 508, 8, 0, 0, 0, 0, 0, 0, 0, 519, 534, 494, 509, 529, 456, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 460, 518, 512, 487, 532, 503, 504, 521, 6, 0, 0, 0, 0, 0, 0, 0, 564, 526, 502, 532, 505, 502, 462, 525, 15, 13, 0, 0, 0, 0, 0, 0, 515, 473, 532, 503, 502, 559, 529, 450, 0, 0, 0, 0, 0, 0, 0, 0, 539, 497, 516, 485, 522, 509, 487, 509, 24, 21, 0, 0, 0, 0, 0, 0, 505, 484, 495, 538, 512, 514, 506, 525, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4073, 4086, 4043, 4146, 4063, 4109, 4125] avg=4096.0000 max=4146 max_rank=4 min=4043 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[518, 535, 541, 533, 502, 498, 502, 524, 0, 0, 0, 0, 0, 0, 0, 0, 527, 508, 488, 464, 500, 504, 520, 512, 3, 0, 0, 0, 0, 0, 0, 0, 544, 530, 511, 492, 496, 431, 525, 521, 0, 0, 0, 0, 0, 0, 0, 0, 521, 503, 519, 492, 510, 511, 477, 503, 2, 0, 0, 0, 0, 0, 0, 0, 513, 516, 515, 532, 546, 518, 509, 518, 13, 9, 0, 0, 0, 0, 0, 0, 481, 498, 498, 499, 520, 550, 554, 566, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 507, 513, 504, 503, 500, 473, 20, 15, 0, 0, 0, 0, 0, 0, 531, 484, 497, 489, 504, 512, 545, 501, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4026, 4050, 4038, 4189, 4166, 4044, 4102] avg=4096.0000 max=4189 max_rank=4 min=4026 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16501 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 3968, 4096, 4096, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 3840, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4096, 4352, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 3968, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[495, 524, 461, 492, 518, 490, 534, 536, 529, 546, 549, 460, 490, 519, 534, 564, 555, 534, 512, 532, 491, 504, 509, 517, 520, 532, 531, 535, 490, 494, 505, 516, 508, 518, 499, 515, 493, 489, 529, 523, 499, 488, 494, 515, 499, 521, 522, 507, 498, 489, 458, 513, 490, 509, 512, 486, 566, 516, 506, 502, 521, 551, 500, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[519, 518, 529, 526, 477, 538, 524, 482, 517, 493, 557, 509, 481, 526, 528, 532, 499, 520, 523, 505, 508, 539, 509, 550, 517, 512, 527, 515, 506, 504, 493, 521, 471, 484, 544, 518, 501, 474, 529, 491, 500, 533, 507, 528, 488, 500, 543, 513, 544, 525, 511, 498, 482, 492, 507, 508, 528, 500, 473, 509, 486, 516, 530, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[514, 505, 531, 558, 505, 496, 516, 524, 531, 537, 492, 510, 502, 476, 484, 527, 523, 511, 516, 521, 530, 534, 517, 525, 513, 454, 547, 493, 452, 498, 503, 548, 559, 518, 517, 498, 528, 490, 512, 501, 502, 540, 536, 532, 539, 517, 526, 519, 541, 496, 530, 473, 512, 505, 492, 527, 510, 515, 483, 482, 509, 508, 487, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[543, 517, 491, 496, 511, 520, 534, 528, 516, 504, 507, 485, 509, 516, 507, 517, 480, 503, 516, 550, 545, 547, 508, 529, 475, 485, 557, 533, 522, 497, 509, 516, 538, 522, 501, 461, 477, 544, 543, 538, 455, 548, 509, 504, 541, 485, 531, 490, 508, 516, 509, 499, 551, 511, 483, 501, 522, 507, 495, 517, 504, 468, 512, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[519, 508, 503, 531, 486, 503, 524, 494, 523, 500, 516, 527, 539, 523, 525, 507, 493, 501, 530, 530, 514, 518, 519, 469, 502, 511, 538, 491, 477, 545, 512, 523, 521, 537, 512, 458, 515, 489, 498, 512, 516, 426, 497, 504, 506, 548, 492, 527, 517, 499, 537, 502, 510, 523, 520, 548, 554, 517, 504, 501, 539, 502, 516, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[500, 501, 556, 507, 563, 479, 496, 527, 525, 550, 508, 524, 522, 534, 488, 484, 480, 500, 511, 504, 526, 524, 545, 514, 504, 518, 503, 508, 507, 535, 535, 503, 476, 496, 486, 547, 473, 464, 504, 512, 473, 470, 527, 547, 471, 550, 482, 508, 505, 485, 515, 495, 535, 553, 520, 514, 501, 489, 538, 522, 523, 521, 534, 551] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[525, 508, 539, 518, 490, 536, 523, 485, 539, 522, 503, 496, 488, 515, 531, 495, 545, 457, 526, 532, 541, 499, 528, 494, 528, 516, 472, 494, 527, 550, 498, 520, 480, 512, 482, 528, 519, 555, 521, 541, 479, 527, 493, 510, 531, 518, 502, 511, 534, 500, 494, 486, 499, 499, 551, 513, 529, 498, 534, 511, 479, 482, 500, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[470, 467, 537, 477, 538, 492, 514, 503, 514, 518, 540, 515, 508, 529, 528, 532, 529, 542, 542, 506, 504, 502, 479, 521, 478, 500, 498, 480, 503, 517, 507, 492, 488, 543, 495, 507, 513, 498, 480, 515, 492, 522, 512, 532, 526, 525, 513, 501, 522, 555, 504, 517, 524, 531, 490, 569, 527, 505, 507, 522, 516, 519, 492, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[519, 508, 503, 531, 486, 503, 524, 494, 0, 0, 0, 0, 0, 0, 0, 0, 494, 500, 516, 527, 539, 523, 525, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 530, 491, 514, 518, 519, 469, 0, 0, 0, 0, 0, 0, 0, 0, 502, 511, 538, 491, 477, 545, 512, 523, 11, 7, 0, 0, 0, 0, 0, 0, 521, 537, 512, 458, 515, 489, 498, 512, 32, 0, 0, 0, 0, 0, 0, 0, 516, 426, 497, 504, 506, 548, 492, 527, 12, 0, 0, 0, 0, 0, 0, 0, 517, 499, 537, 502, 510, 523, 520, 548, 2, 0, 0, 0, 0, 0, 0, 0, 554, 517, 504, 501, 539, 502, 516, 520, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4131, 4035, 4117, 4074, 4028, 4158, 4157] avg=4096.0000 max=4158 max_rank=6 min=4028 min_rank=5 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[514, 505, 531, 558, 505, 496, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 501, 537, 492, 510, 502, 476, 484, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 511, 516, 482, 530, 534, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 513, 454, 547, 493, 452, 498, 503, 548, 12, 7, 0, 0, 0, 0, 0, 0, 559, 518, 517, 498, 528, 490, 512, 501, 32, 0, 0, 0, 0, 0, 0, 0, 502, 540, 536, 532, 539, 517, 526, 519, 12, 0, 0, 0, 0, 0, 0, 0, 541, 496, 530, 473, 512, 505, 492, 527, 2, 0, 0, 0, 0, 0, 0, 0, 510, 515, 483, 482, 509, 508, 487, 471, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4149, 4029, 4138, 4027, 4155, 4223, 4078, 3969] avg=4096.0000 max=4223 max_rank=5 min=3969 min_rank=7 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16425 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[519, 518, 529, 526, 477, 538, 524, 482, 0, 0, 0, 0, 0, 0, 0, 0, 488, 493, 557, 509, 481, 526, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 499, 520, 523, 467, 508, 539, 509, 550, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 527, 515, 506, 504, 493, 521, 11, 7, 0, 0, 0, 0, 0, 0, 471, 484, 544, 518, 501, 474, 529, 491, 31, 0, 0, 0, 0, 0, 0, 0, 500, 533, 507, 528, 488, 500, 543, 513, 12, 0, 0, 0, 0, 0, 0, 0, 544, 525, 511, 498, 482, 492, 507, 508, 2, 0, 0, 0, 0, 0, 0, 0, 528, 500, 473, 509, 486, 516, 530, 531, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4114, 4115, 4113, 4043, 4124, 4069, 4077] avg=4096.0000 max=4124 max_rank=5 min=4043 min_rank=4 imbalance_ratio=1.006836 max_load_reduction=28644 max_load_reduction_pct=0.874146 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[495, 524, 461, 492, 518, 490, 534, 536, 0, 0, 0, 0, 0, 0, 0, 0, 500, 546, 549, 460, 490, 519, 534, 564, 0, 0, 0, 0, 0, 0, 0, 0, 555, 534, 512, 493, 491, 504, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 520, 532, 531, 535, 490, 494, 505, 516, 11, 7, 0, 0, 0, 0, 0, 0, 508, 518, 499, 515, 493, 489, 529, 523, 32, 0, 0, 0, 0, 0, 0, 0, 499, 488, 494, 515, 499, 521, 522, 507, 12, 0, 0, 0, 0, 0, 0, 0, 498, 489, 458, 513, 490, 509, 512, 486, 2, 0, 0, 0, 0, 0, 0, 0, 566, 516, 506, 502, 521, 551, 500, 514, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4050, 4162, 4115, 4141, 4106, 4057, 3957, 4180] avg=4096.0000 max=4180 max_rank=7 min=3957 min_rank=6 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16300 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[543, 517, 491, 496, 511, 520, 534, 528, 0, 0, 0, 0, 0, 0, 0, 0, 487, 504, 507, 485, 509, 516, 507, 517, 0, 0, 0, 0, 0, 0, 0, 0, 480, 503, 516, 509, 545, 547, 508, 529, 0, 0, 0, 0, 0, 0, 0, 0, 475, 485, 557, 533, 522, 497, 509, 516, 11, 8, 0, 0, 0, 0, 0, 0, 538, 522, 501, 461, 477, 544, 543, 538, 33, 0, 0, 0, 0, 0, 0, 0, 455, 548, 509, 504, 541, 485, 531, 490, 12, 0, 0, 0, 0, 0, 0, 0, 508, 516, 509, 499, 551, 511, 483, 501, 2, 0, 0, 0, 0, 0, 0, 0, 522, 507, 495, 517, 504, 468, 512, 505, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4032, 4137, 4113, 4157, 4075, 4080, 4034] avg=4096.0000 max=4157 max_rank=4 min=4032 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[525, 508, 539, 518, 490, 536, 523, 485, 0, 0, 0, 0, 0, 0, 0, 0, 509, 522, 503, 496, 488, 515, 531, 495, 0, 0, 0, 0, 0, 0, 0, 0, 545, 457, 526, 493, 541, 499, 528, 494, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 472, 494, 527, 550, 498, 520, 12, 7, 0, 0, 0, 0, 0, 0, 480, 512, 482, 528, 519, 555, 521, 541, 32, 0, 0, 0, 0, 0, 0, 0, 479, 527, 493, 510, 531, 518, 502, 511, 12, 0, 0, 0, 0, 0, 0, 0, 534, 500, 494, 486, 499, 499, 551, 513, 2, 0, 0, 0, 0, 0, 0, 0, 529, 498, 534, 511, 479, 482, 500, 510, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4059, 4083, 4124, 4170, 4083, 4078, 4047] avg=4096.0000 max=4170 max_rank=4 min=4047 min_rank=7 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 501, 556, 507, 563, 479, 496, 527, 0, 0, 0, 0, 0, 0, 0, 0, 496, 550, 508, 524, 522, 534, 488, 484, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 511, 466, 526, 524, 545, 514, 0, 0, 0, 0, 0, 0, 0, 0, 504, 518, 503, 508, 507, 535, 535, 503, 11, 7, 0, 0, 0, 0, 0, 0, 476, 496, 486, 547, 473, 464, 504, 512, 31, 0, 0, 0, 0, 0, 0, 0, 473, 470, 527, 547, 471, 550, 482, 508, 12, 0, 0, 0, 0, 0, 0, 0, 505, 485, 515, 495, 535, 553, 520, 514, 2, 0, 0, 0, 0, 0, 0, 0, 501, 489, 538, 522, 523, 521, 534, 551, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4106, 4066, 4131, 3989, 4040, 4124, 4183] avg=4096.0000 max=4183 max_rank=7 min=3989 min_rank=4 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[470, 467, 537, 477, 538, 492, 511, 503, 0, 0, 0, 0, 0, 0, 0, 0, 465, 518, 540, 515, 508, 529, 528, 532, 0, 0, 0, 0, 0, 0, 0, 0, 529, 542, 542, 460, 504, 502, 479, 521, 0, 0, 0, 0, 0, 0, 0, 0, 478, 500, 498, 480, 503, 517, 507, 492, 16, 11, 0, 0, 0, 0, 0, 0, 488, 543, 495, 507, 513, 498, 480, 515, 35, 0, 0, 0, 0, 0, 0, 0, 492, 522, 512, 532, 526, 525, 513, 501, 15, 0, 0, 0, 0, 0, 0, 0, 522, 555, 504, 517, 524, 531, 490, 569, 9, 3, 0, 0, 0, 0, 0, 0, 527, 505, 507, 522, 516, 519, 492, 524, 9, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 4135, 4079, 4002, 4074, 4138, 4224, 4121] avg=4096.0000 max=4224 max_rank=6 min=3995 min_rank=0 imbalance_ratio=1.031250 max_load_reduction=28544 max_load_reduction_pct=0.871094 moved_to_echo=16557 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[515, 513, 504, 486, 509, 471, 497, 547, 529, 544, 516, 513, 531, 505, 510, 532, 501, 514, 535, 474, 519, 518, 516, 524, 499, 512, 522, 466, 509, 517, 511, 550, 480, 541, 491, 493, 555, 490, 504, 526, 525, 550, 563, 533, 474, 501, 519, 495, 483, 499, 529, 501, 520, 473, 469, 504, 500, 524, 490, 529, 509, 531, 532, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[515, 483, 521, 512, 540, 482, 507, 514, 537, 516, 510, 549, 538, 520, 516, 506, 505, 555, 505, 569, 511, 515, 517, 474, 538, 491, 495, 545, 508, 538, 472, 543, 518, 543, 497, 508, 539, 464, 513, 551, 508, 488, 493, 528, 497, 494, 513, 510, 504, 496, 495, 499, 465, 490, 487, 524, 531, 534, 540, 505, 469, 481, 503, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[528, 499, 498, 485, 540, 495, 545, 491, 507, 512, 548, 522, 478, 459, 512, 542, 465, 497, 540, 490, 521, 569, 519, 491, 510, 473, 538, 561, 505, 474, 506, 541, 550, 514, 528, 529, 476, 524, 549, 497, 514, 521, 523, 512, 514, 511, 500, 525, 524, 484, 522, 504, 509, 503, 536, 497, 536, 482, 481, 487, 476, 540, 512, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[506, 536, 534, 541, 494, 545, 489, 551, 541, 507, 550, 480, 476, 524, 550, 498, 539, 464, 514, 535, 513, 531, 510, 517, 488, 480, 498, 528, 488, 547, 512, 512, 529, 513, 472, 520, 515, 513, 517, 491, 497, 507, 553, 459, 518, 521, 516, 494, 521, 532, 549, 485, 484, 461, 502, 539, 532, 531, 460, 505, 517, 515, 502, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[522, 529, 506, 498, 487, 495, 538, 512, 481, 533, 545, 490, 514, 552, 508, 547, 487, 510, 493, 518, 561, 505, 495, 524, 487, 521, 505, 512, 496, 529, 456, 495, 473, 481, 515, 498, 552, 526, 511, 531, 494, 508, 553, 495, 537, 468, 463, 498, 536, 525, 539, 504, 509, 552, 504, 522, 521, 524, 486, 532, 509, 525, 509, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[551, 544, 482, 511, 541, 539, 506, 543, 508, 560, 529, 512, 523, 500, 505, 492, 540, 499, 529, 500, 495, 498, 495, 536, 537, 545, 515, 511, 553, 489, 496, 516, 468, 488, 493, 532, 512, 525, 526, 486, 535, 511, 517, 466, 528, 516, 505, 479, 530, 539, 509, 512, 514, 512, 497, 508, 492, 527, 464, 489, 564, 492, 445, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[519, 518, 551, 507, 496, 508, 502, 544, 524, 521, 540, 496, 487, 521, 528, 517, 518, 491, 532, 483, 492, 504, 489, 503, 498, 521, 525, 527, 490, 520, 489, 504, 499, 518, 528, 510, 494, 528, 533, 534, 541, 478, 510, 498, 498, 522, 456, 496, 521, 510, 523, 495, 503, 507, 512, 509, 527, 494, 485, 534, 536, 556, 494, 554] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[490, 512, 500, 488, 504, 547, 539, 524, 496, 512, 494, 524, 547, 474, 539, 505, 551, 531, 496, 560, 521, 444, 483, 558, 500, 500, 518, 524, 479, 486, 473, 471, 484, 534, 500, 523, 518, 533, 499, 546, 539, 500, 535, 498, 540, 503, 510, 516, 511, 536, 512, 548, 511, 524, 519, 500, 516, 505, 489, 466, 541, 479, 538, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 483, 521, 512, 540, 482, 507, 494, 0, 0, 0, 0, 0, 0, 0, 0, 537, 516, 471, 549, 538, 520, 516, 506, 0, 0, 0, 0, 0, 0, 0, 0, 505, 555, 501, 569, 511, 515, 517, 474, 0, 0, 0, 0, 0, 0, 0, 0, 538, 491, 495, 545, 508, 538, 472, 543, 24, 0, 0, 0, 0, 0, 0, 0, 518, 543, 497, 508, 539, 464, 513, 541, 0, 0, 0, 0, 0, 0, 0, 0, 508, 488, 493, 528, 497, 494, 513, 510, 15, 5, 0, 0, 0, 0, 0, 0, 504, 496, 495, 499, 465, 490, 487, 524, 15, 0, 0, 0, 0, 0, 0, 0, 531, 534, 540, 505, 469, 481, 503, 534, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4054, 4153, 4147, 4154, 4123, 4051, 3975, 4111] avg=4096.0000 max=4154 max_rank=3 min=3975 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16260 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[515, 513, 504, 486, 509, 471, 497, 525, 0, 0, 0, 0, 0, 0, 0, 0, 529, 544, 477, 513, 531, 505, 510, 532, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 530, 474, 519, 518, 516, 524, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 522, 466, 509, 517, 511, 550, 24, 0, 0, 0, 0, 0, 0, 0, 480, 541, 491, 493, 555, 490, 504, 516, 0, 0, 0, 0, 0, 0, 0, 0, 525, 550, 563, 533, 474, 501, 519, 495, 15, 6, 0, 0, 0, 0, 0, 0, 483, 499, 529, 501, 520, 473, 469, 504, 16, 0, 0, 0, 0, 0, 0, 0, 500, 524, 490, 529, 509, 531, 532, 526, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4141, 4096, 4110, 4070, 4181, 3994, 4156] avg=4096.0000 max=4181 max_rank=5 min=3994 min_rank=6 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[551, 544, 482, 511, 541, 539, 506, 521, 0, 0, 0, 0, 0, 0, 0, 0, 508, 560, 488, 512, 523, 500, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 540, 499, 524, 500, 495, 498, 495, 536, 0, 0, 0, 0, 0, 0, 0, 0, 537, 545, 515, 511, 553, 489, 496, 516, 25, 0, 0, 0, 0, 0, 0, 0, 468, 488, 493, 532, 512, 525, 526, 477, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 517, 466, 528, 516, 505, 479, 16, 6, 0, 0, 0, 0, 0, 0, 530, 539, 509, 512, 514, 512, 497, 508, 16, 0, 0, 0, 0, 0, 0, 0, 492, 527, 464, 489, 564, 492, 445, 487, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4195, 4088, 4087, 4187, 4021, 4079, 4137, 3974] avg=4096.0000 max=4195 max_rank=0 min=3974 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16211 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[528, 499, 498, 485, 540, 495, 545, 472, 0, 0, 0, 0, 0, 0, 0, 0, 507, 512, 506, 522, 478, 459, 512, 542, 0, 0, 0, 0, 0, 0, 0, 0, 465, 497, 535, 490, 521, 569, 519, 491, 0, 0, 0, 0, 0, 0, 0, 0, 510, 473, 538, 561, 505, 474, 506, 541, 26, 0, 0, 0, 0, 0, 0, 0, 550, 514, 528, 529, 476, 524, 549, 488, 0, 0, 0, 0, 0, 0, 0, 0, 514, 521, 523, 512, 514, 511, 500, 525, 16, 5, 0, 0, 0, 0, 0, 0, 524, 484, 522, 504, 509, 503, 536, 497, 14, 0, 0, 0, 0, 0, 0, 0, 536, 482, 481, 487, 476, 540, 512, 527, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4038, 4087, 4134, 4158, 4141, 4093, 4055] avg=4096.0000 max=4158 max_rank=4 min=4038 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 536, 534, 541, 494, 545, 489, 529, 0, 0, 0, 0, 0, 0, 0, 0, 541, 507, 507, 480, 476, 524, 550, 498, 0, 0, 0, 0, 0, 0, 0, 0, 539, 464, 510, 535, 513, 531, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 488, 480, 498, 528, 488, 547, 512, 512, 26, 0, 0, 0, 0, 0, 0, 0, 529, 513, 472, 520, 515, 513, 517, 482, 0, 0, 0, 0, 0, 0, 0, 0, 497, 507, 553, 459, 518, 521, 516, 494, 17, 6, 0, 0, 0, 0, 0, 0, 521, 532, 549, 485, 484, 461, 502, 539, 16, 0, 0, 0, 0, 0, 0, 0, 532, 531, 460, 505, 517, 515, 502, 500, 9, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4083, 4119, 4079, 4061, 4088, 4089, 4075] avg=4096.0000 max=4174 max_rank=0 min=4061 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16313 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[490, 512, 500, 488, 504, 547, 539, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 512, 456, 524, 547, 474, 539, 505, 0, 0, 0, 0, 0, 0, 0, 0, 551, 531, 492, 560, 521, 444, 483, 558, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 518, 524, 479, 486, 473, 471, 23, 0, 0, 0, 0, 0, 0, 0, 484, 534, 500, 523, 518, 533, 499, 536, 0, 0, 0, 0, 0, 0, 0, 0, 539, 500, 535, 498, 540, 503, 510, 516, 15, 5, 0, 0, 0, 0, 0, 0, 511, 536, 512, 548, 511, 524, 519, 500, 15, 0, 0, 0, 0, 0, 0, 0, 516, 505, 489, 466, 541, 479, 538, 505, 10, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4053, 4140, 3974, 4127, 4161, 4176, 4053] avg=4096.0000 max=4176 max_rank=6 min=3974 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16517 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[522, 529, 506, 498, 487, 495, 538, 481, 0, 0, 0, 0, 0, 0, 0, 0, 481, 533, 494, 490, 514, 552, 508, 547, 0, 0, 0, 0, 0, 0, 0, 0, 487, 510, 485, 518, 561, 505, 495, 524, 0, 0, 0, 0, 0, 0, 0, 0, 487, 521, 505, 512, 496, 529, 456, 495, 30, 0, 0, 0, 0, 0, 0, 0, 473, 481, 515, 498, 552, 526, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 494, 508, 553, 495, 537, 468, 463, 498, 21, 9, 0, 0, 0, 0, 0, 0, 536, 525, 539, 504, 509, 552, 504, 522, 17, 0, 0, 0, 0, 0, 0, 0, 521, 524, 486, 532, 509, 525, 509, 517, 13, 8, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4119, 4085, 4031, 4074, 4046, 4208, 4149] avg=4096.0000 max=4208 max_rank=6 min=4031 min_rank=3 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 518, 551, 507, 496, 508, 502, 522, 0, 0, 0, 0, 0, 0, 0, 0, 524, 521, 499, 496, 487, 521, 528, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 491, 527, 483, 492, 504, 489, 503, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 525, 527, 490, 520, 489, 504, 25, 0, 0, 0, 0, 0, 0, 0, 499, 518, 528, 510, 494, 528, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 541, 478, 510, 498, 498, 522, 456, 496, 16, 6, 0, 0, 0, 0, 0, 0, 521, 510, 523, 495, 503, 507, 512, 509, 16, 0, 0, 0, 0, 0, 0, 0, 527, 494, 485, 534, 536, 556, 494, 554, 10, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4093, 4007, 4099, 4134, 4021, 4096, 4195] avg=4096.0000 max=4195 max_rank=7 min=4007 min_rank=2 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4352, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 3968, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 3968, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 3968, 4096, 4224, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[543, 530, 541, 488, 537, 536, 503, 501, 506, 512, 508, 523, 521, 540, 510, 544, 505, 548, 521, 518, 524, 499, 547, 513, 523, 485, 495, 499, 492, 523, 526, 514, 509, 516, 534, 506, 474, 508, 481, 495, 511, 526, 499, 512, 513, 505, 505, 531, 499, 483, 486, 475, 489, 529, 555, 490, 521, 507, 513, 516, 512, 505, 492, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[488, 523, 545, 560, 542, 475, 517, 513, 497, 545, 550, 494, 513, 534, 512, 497, 513, 477, 531, 496, 526, 506, 476, 463, 491, 483, 526, 475, 531, 495, 505, 522, 519, 514, 517, 537, 526, 482, 507, 519, 528, 544, 493, 536, 536, 518, 491, 516, 481, 502, 537, 525, 534, 524, 534, 523, 489, 518, 477, 523, 500, 496, 500, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[511, 471, 497, 540, 491, 511, 511, 507, 502, 561, 503, 492, 493, 489, 505, 503, 530, 488, 488, 485, 522, 517, 508, 504, 526, 506, 513, 539, 535, 498, 528, 503, 516, 544, 524, 528, 524, 539, 503, 517, 516, 529, 523, 486, 505, 500, 480, 539, 527, 526, 516, 519, 501, 515, 475, 501, 551, 546, 492, 542, 524, 499, 494, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[481, 534, 505, 539, 487, 523, 522, 479, 467, 494, 508, 516, 535, 529, 532, 496, 521, 508, 486, 493, 509, 528, 465, 489, 463, 494, 507, 536, 495, 534, 519, 508, 548, 530, 502, 492, 483, 548, 539, 530, 527, 544, 518, 578, 487, 528, 526, 512, 499, 496, 508, 513, 529, 497, 569, 509, 485, 481, 526, 492, 523, 483, 525, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[508, 511, 516, 535, 500, 530, 498, 495, 511, 493, 504, 484, 538, 514, 516, 484, 524, 553, 497, 491, 542, 465, 478, 520, 534, 494, 474, 524, 566, 520, 515, 535, 505, 506, 514, 479, 486, 535, 543, 513, 594, 504, 480, 530, 510, 530, 512, 525, 485, 487, 532, 524, 514, 533, 502, 486, 520, 483, 519, 508, 498, 497, 520, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[515, 531, 519, 515, 500, 516, 527, 498, 516, 474, 500, 528, 504, 524, 520, 499, 570, 494, 474, 475, 499, 544, 531, 518, 518, 490, 546, 490, 505, 514, 505, 516, 521, 532, 499, 500, 478, 508, 500, 523, 524, 541, 521, 495, 519, 480, 522, 480, 538, 516, 534, 498, 468, 520, 497, 517, 498, 529, 527, 536, 522, 512, 513, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[512, 538, 539, 490, 517, 514, 472, 533, 520, 546, 509, 501, 471, 540, 535, 505, 507, 541, 483, 475, 514, 551, 515, 519, 507, 539, 512, 480, 498, 497, 547, 532, 522, 517, 529, 504, 510, 503, 518, 484, 499, 497, 526, 500, 501, 522, 512, 531, 554, 517, 496, 481, 504, 495, 502, 514, 502, 509, 519, 493, 515, 531, 506, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[514, 506, 513, 496, 485, 501, 499, 509, 501, 477, 508, 544, 527, 496, 512, 535, 511, 525, 497, 512, 502, 500, 508, 523, 505, 494, 567, 537, 530, 540, 504, 527, 530, 485, 528, 513, 522, 509, 510, 536, 521, 489, 509, 472, 535, 524, 525, 501, 485, 560, 486, 540, 512, 503, 520, 511, 507, 497, 494, 512, 507, 552, 452, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[543, 530, 537, 488, 537, 536, 503, 501, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 508, 523, 521, 540, 510, 544, 0, 0, 0, 0, 0, 0, 0, 0, 505, 548, 521, 518, 524, 499, 547, 513, 24, 0, 0, 0, 0, 0, 0, 0, 523, 485, 495, 499, 492, 523, 526, 505, 0, 0, 0, 0, 0, 0, 0, 0, 497, 516, 534, 506, 474, 508, 481, 495, 0, 0, 0, 0, 0, 0, 0, 0, 485, 526, 499, 512, 513, 505, 505, 531, 0, 0, 0, 0, 0, 0, 0, 0, 499, 483, 486, 475, 489, 529, 555, 490, 4, 4, 0, 0, 0, 0, 0, 0, 521, 507, 513, 516, 512, 505, 492, 496, 12, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4175, 4164, 4199, 4048, 4011, 4076, 4014, 4081] avg=4096.0000 max=4199 max_rank=2 min=4011 min_rank=4 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16182 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 471, 493, 540, 491, 511, 511, 507, 0, 0, 0, 0, 0, 0, 0, 0, 502, 561, 503, 492, 493, 489, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 530, 488, 488, 485, 522, 517, 508, 504, 25, 0, 0, 0, 0, 0, 0, 0, 526, 506, 513, 539, 535, 498, 528, 494, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 524, 528, 524, 539, 503, 517, 0, 0, 0, 0, 0, 0, 0, 0, 489, 529, 523, 486, 505, 500, 480, 539, 0, 0, 0, 0, 0, 0, 0, 0, 527, 526, 516, 519, 501, 515, 475, 501, 4, 4, 0, 0, 0, 0, 0, 0, 551, 546, 492, 542, 524, 499, 494, 490, 12, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4048, 4067, 4139, 4183, 4051, 4088, 4157] avg=4096.0000 max=4183 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16479 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 523, 541, 560, 542, 475, 517, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 545, 550, 494, 513, 534, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 513, 477, 531, 496, 526, 506, 476, 463, 25, 0, 0, 0, 0, 0, 0, 0, 491, 483, 526, 475, 531, 495, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 506, 514, 517, 537, 526, 482, 507, 519, 0, 0, 0, 0, 0, 0, 0, 0, 501, 544, 493, 536, 536, 518, 491, 516, 0, 0, 0, 0, 0, 0, 0, 0, 481, 502, 537, 525, 534, 524, 534, 523, 4, 4, 0, 0, 0, 0, 0, 0, 489, 518, 477, 523, 500, 496, 500, 501, 13, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4142, 4013, 4019, 4108, 4135, 4168, 4024] avg=4096.0000 max=4168 max_rank=6 min=4013 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[481, 534, 501, 539, 487, 523, 522, 479, 0, 0, 0, 0, 0, 0, 0, 0, 467, 494, 508, 516, 535, 529, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 521, 508, 486, 493, 509, 528, 465, 489, 25, 0, 0, 0, 0, 0, 0, 0, 463, 494, 507, 536, 495, 534, 519, 499, 0, 0, 0, 0, 0, 0, 0, 0, 535, 530, 502, 492, 483, 548, 539, 530, 0, 0, 0, 0, 0, 0, 0, 0, 500, 544, 518, 578, 487, 528, 526, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 496, 508, 513, 529, 497, 569, 509, 4, 4, 0, 0, 0, 0, 0, 0, 485, 481, 526, 492, 523, 483, 525, 539, 13, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4077, 4024, 4047, 4159, 4193, 4128, 4074] avg=4096.0000 max=4193 max_rank=5 min=4024 min_rank=2 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[508, 511, 512, 535, 500, 530, 498, 495, 0, 0, 0, 0, 0, 0, 0, 0, 511, 493, 504, 484, 538, 514, 516, 484, 0, 0, 0, 0, 0, 0, 0, 0, 524, 553, 497, 491, 542, 465, 478, 520, 28, 0, 0, 0, 0, 0, 0, 0, 534, 494, 474, 524, 566, 520, 515, 526, 0, 0, 0, 0, 0, 0, 0, 0, 493, 506, 514, 479, 486, 535, 543, 513, 0, 0, 0, 0, 0, 0, 0, 0, 564, 504, 480, 530, 510, 530, 512, 525, 0, 0, 0, 0, 0, 0, 0, 0, 485, 487, 532, 524, 514, 533, 502, 486, 4, 4, 0, 0, 0, 0, 0, 0, 520, 483, 519, 508, 498, 497, 520, 525, 12, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4089, 4044, 4098, 4153, 4069, 4155, 4071, 4089] avg=4096.0000 max=4155 max_rank=5 min=4044 min_rank=1 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[514, 506, 509, 496, 485, 501, 499, 509, 0, 0, 0, 0, 0, 0, 0, 0, 501, 477, 508, 544, 527, 496, 512, 535, 0, 0, 0, 0, 0, 0, 0, 0, 511, 525, 497, 512, 502, 500, 508, 523, 25, 0, 0, 0, 0, 0, 0, 0, 505, 494, 567, 537, 530, 540, 504, 518, 0, 0, 0, 0, 0, 0, 0, 0, 517, 485, 528, 513, 522, 509, 510, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 489, 509, 472, 535, 524, 525, 501, 0, 0, 0, 0, 0, 0, 0, 0, 485, 560, 486, 540, 512, 503, 520, 511, 4, 4, 0, 0, 0, 0, 0, 0, 507, 497, 494, 512, 507, 552, 452, 516, 13, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4100, 4103, 4195, 4120, 4049, 4125, 4057] avg=4096.0000 max=4195 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[515, 531, 512, 515, 500, 516, 527, 498, 0, 0, 0, 0, 0, 0, 0, 0, 516, 474, 500, 528, 504, 524, 520, 499, 1, 0, 0, 0, 0, 0, 0, 0, 570, 494, 474, 475, 499, 544, 531, 518, 30, 0, 0, 0, 0, 0, 0, 0, 518, 490, 546, 490, 505, 514, 505, 496, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 499, 500, 478, 508, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 489, 541, 521, 495, 519, 480, 522, 480, 0, 0, 0, 0, 0, 0, 0, 0, 538, 516, 534, 498, 468, 520, 497, 517, 9, 6, 0, 0, 0, 0, 0, 0, 498, 529, 527, 536, 522, 512, 513, 525, 17, 11, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4066, 4135, 4064, 4044, 4047, 4103, 4195] avg=4096.0000 max=4195 max_rank=7 min=4044 min_rank=4 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 538, 535, 490, 517, 514, 472, 533, 0, 0, 0, 0, 0, 0, 0, 0, 520, 546, 509, 501, 471, 540, 535, 505, 0, 0, 0, 0, 0, 0, 0, 0, 507, 541, 483, 475, 514, 551, 515, 519, 24, 0, 0, 0, 0, 0, 0, 0, 507, 539, 512, 480, 498, 497, 547, 523, 0, 0, 0, 0, 0, 0, 0, 0, 509, 517, 529, 504, 510, 503, 518, 484, 0, 0, 0, 0, 0, 0, 0, 0, 473, 497, 526, 500, 501, 522, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 554, 517, 496, 481, 504, 495, 502, 514, 4, 4, 0, 0, 0, 0, 0, 0, 502, 509, 519, 493, 515, 531, 506, 496, 13, 5, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4127, 4129, 4103, 4074, 4062, 4071, 4091] avg=4096.0000 max=4129 max_rank=2 min=4062 min_rank=5 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[540, 511, 511, 530, 523, 532, 504, 506, 506, 517, 503, 525, 533, 518, 516, 542, 515, 531, 523, 523, 489, 473, 535, 534, 523, 527, 498, 536, 514, 516, 539, 497, 501, 553, 497, 529, 492, 543, 510, 551, 546, 473, 473, 479, 538, 493, 499, 531, 528, 466, 482, 522, 513, 518, 477, 510, 526, 482, 482, 510, 480, 502, 474, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[508, 506, 504, 526, 496, 470, 501, 525, 473, 486, 484, 494, 488, 511, 507, 503, 541, 523, 501, 527, 501, 562, 528, 484, 496, 513, 504, 519, 558, 530, 531, 535, 537, 519, 505, 535, 550, 523, 513, 525, 485, 524, 509, 488, 466, 509, 516, 520, 528, 534, 515, 486, 503, 491, 549, 512, 494, 502, 537, 502, 537, 510, 475, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[526, 544, 531, 519, 510, 526, 502, 525, 518, 507, 512, 519, 513, 529, 493, 518, 527, 492, 562, 501, 524, 545, 478, 508, 481, 491, 535, 542, 526, 544, 537, 490, 529, 457, 507, 501, 473, 548, 530, 533, 523, 529, 467, 528, 512, 506, 475, 494, 504, 525, 517, 534, 490, 488, 491, 504, 491, 507, 504, 480, 547, 474, 488, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[507, 535, 508, 479, 505, 488, 551, 537, 534, 486, 525, 488, 487, 526, 490, 514, 499, 538, 504, 516, 490, 516, 469, 548, 545, 516, 532, 541, 485, 526, 501, 526, 499, 516, 455, 500, 523, 540, 509, 487, 528, 534, 477, 527, 534, 557, 532, 533, 506, 534, 508, 486, 525, 498, 515, 477, 492, 495, 486, 502, 548, 483, 537, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[556, 540, 514, 501, 519, 533, 525, 509, 504, 544, 533, 498, 464, 520, 540, 534, 511, 517, 511, 523, 526, 500, 489, 529, 529, 509, 515, 498, 541, 513, 486, 488, 478, 491, 523, 527, 491, 514, 497, 494, 506, 493, 480, 473, 514, 505, 532, 502, 523, 498, 514, 557, 494, 510, 488, 483, 521, 512, 528, 533, 500, 533, 491, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[503, 528, 507, 529, 507, 510, 523, 532, 489, 547, 484, 509, 515, 507, 496, 523, 486, 478, 539, 529, 572, 511, 533, 537, 530, 534, 500, 521, 512, 484, 550, 495, 526, 515, 500, 492, 490, 486, 532, 544, 526, 521, 482, 532, 495, 529, 504, 530, 464, 513, 501, 491, 453, 537, 494, 504, 513, 545, 517, 476, 537, 497, 487, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[469, 514, 525, 522, 537, 529, 548, 537, 475, 523, 515, 524, 508, 526, 492, 477, 513, 499, 492, 535, 524, 514, 492, 556, 511, 506, 532, 512, 505, 471, 535, 485, 525, 516, 493, 474, 504, 507, 513, 534, 511, 509, 536, 547, 539, 510, 532, 513, 488, 460, 471, 513, 498, 522, 541, 501, 510, 511, 509, 524, 510, 558, 479, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[499, 536, 484, 513, 498, 503, 525, 543, 512, 543, 488, 492, 504, 524, 527, 516, 506, 514, 494, 529, 523, 510, 526, 521, 529, 503, 513, 504, 549, 501, 536, 491, 484, 516, 481, 522, 492, 517, 505, 501, 512, 519, 510, 539, 491, 532, 508, 516, 501, 531, 526, 492, 518, 510, 504, 521, 509, 497, 536, 495, 523, 494, 510, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 506, 504, 526, 496, 470, 501, 484, 0, 0, 0, 0, 0, 0, 0, 0, 473, 486, 484, 494, 488, 511, 507, 503, 18, 0, 0, 0, 0, 0, 0, 0, 541, 523, 501, 527, 501, 562, 528, 453, 0, 0, 0, 0, 0, 0, 0, 0, 496, 513, 504, 519, 558, 530, 500, 535, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 505, 535, 550, 523, 513, 525, 11, 0, 0, 0, 0, 0, 0, 0, 485, 524, 509, 488, 466, 509, 516, 520, 1, 0, 0, 0, 0, 0, 0, 0, 528, 534, 515, 486, 503, 491, 549, 512, 41, 8, 0, 0, 0, 0, 0, 0, 494, 502, 537, 502, 537, 510, 475, 534, 23, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3995, 3964, 4136, 4155, 4218, 4018, 4167, 4115] avg=4096.0000 max=4218 max_rank=4 min=3964 min_rank=1 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16518 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[507, 535, 508, 479, 505, 488, 551, 495, 0, 0, 0, 0, 0, 0, 0, 0, 534, 486, 525, 488, 487, 526, 490, 514, 17, 0, 0, 0, 0, 0, 0, 0, 499, 538, 504, 516, 490, 516, 469, 513, 0, 0, 0, 0, 0, 0, 0, 0, 545, 516, 532, 541, 485, 526, 471, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 516, 455, 500, 523, 540, 509, 487, 11, 0, 0, 0, 0, 0, 0, 0, 528, 534, 477, 527, 534, 557, 532, 533, 1, 0, 0, 0, 0, 0, 0, 0, 506, 534, 508, 486, 525, 498, 515, 477, 42, 9, 0, 0, 0, 0, 0, 0, 492, 495, 486, 502, 548, 483, 537, 513, 26, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4067, 4045, 4142, 4040, 4223, 4100, 4083] avg=4096.0000 max=4223 max_rank=5 min=4040 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[540, 511, 511, 530, 523, 532, 504, 466, 0, 0, 0, 0, 0, 0, 0, 0, 506, 517, 503, 525, 533, 518, 516, 542, 19, 0, 0, 0, 0, 0, 0, 0, 515, 531, 523, 523, 489, 473, 535, 500, 0, 0, 0, 0, 0, 0, 0, 0, 523, 527, 498, 536, 514, 516, 506, 497, 0, 0, 0, 0, 0, 0, 0, 0, 501, 553, 497, 529, 492, 543, 510, 551, 12, 0, 0, 0, 0, 0, 0, 0, 546, 473, 473, 479, 538, 493, 499, 531, 1, 0, 0, 0, 0, 0, 0, 0, 528, 466, 482, 522, 513, 518, 477, 510, 40, 9, 0, 0, 0, 0, 0, 0, 526, 482, 482, 510, 480, 502, 474, 498, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4117, 4179, 4089, 4117, 4188, 4033, 4065, 3980] avg=4096.0000 max=4188 max_rank=4 min=3980 min_rank=7 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 528, 507, 529, 507, 510, 523, 490, 0, 0, 0, 0, 0, 0, 0, 0, 489, 547, 484, 509, 515, 507, 496, 523, 19, 0, 0, 0, 0, 0, 0, 0, 486, 478, 539, 529, 572, 511, 533, 503, 0, 0, 0, 0, 0, 0, 0, 0, 530, 534, 500, 521, 512, 484, 517, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 515, 500, 492, 490, 486, 532, 544, 12, 0, 0, 0, 0, 0, 0, 0, 526, 521, 482, 532, 495, 529, 504, 530, 1, 0, 0, 0, 0, 0, 0, 0, 464, 513, 501, 491, 453, 537, 494, 504, 42, 9, 0, 0, 0, 0, 0, 0, 513, 545, 517, 476, 537, 497, 487, 515, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4089, 4151, 4093, 4097, 4120, 4008, 4113] avg=4096.0000 max=4151 max_rank=2 min=4008 min_rank=6 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[526, 544, 531, 519, 510, 526, 502, 484, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 512, 519, 513, 529, 493, 518, 19, 0, 0, 0, 0, 0, 0, 0, 527, 492, 562, 501, 524, 545, 478, 475, 0, 0, 0, 0, 0, 0, 0, 0, 481, 491, 535, 542, 526, 544, 505, 490, 0, 0, 0, 0, 0, 0, 0, 0, 529, 457, 507, 501, 473, 548, 530, 533, 11, 0, 0, 0, 0, 0, 0, 0, 523, 529, 467, 528, 512, 506, 475, 494, 1, 0, 0, 0, 0, 0, 0, 0, 504, 525, 517, 534, 490, 488, 491, 504, 41, 9, 0, 0, 0, 0, 0, 0, 491, 507, 504, 480, 547, 474, 488, 537, 24, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4128, 4104, 4114, 4089, 4035, 4103, 4053] avg=4096.0000 max=4142 max_rank=0 min=4035 min_rank=5 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16280 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[499, 536, 484, 513, 498, 503, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 543, 488, 492, 504, 524, 527, 516, 19, 0, 0, 0, 0, 0, 0, 0, 506, 514, 494, 529, 523, 510, 526, 487, 0, 0, 0, 0, 0, 0, 0, 0, 529, 503, 513, 504, 549, 501, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 484, 516, 481, 522, 492, 517, 505, 501, 11, 0, 0, 0, 0, 0, 0, 0, 512, 519, 510, 539, 491, 532, 508, 516, 1, 0, 0, 0, 0, 0, 0, 0, 501, 531, 526, 492, 518, 510, 504, 521, 43, 9, 0, 0, 0, 0, 0, 0, 509, 497, 536, 495, 523, 494, 510, 500, 25, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4125, 4089, 4094, 4029, 4128, 4155, 4090] avg=4096.0000 max=4155 max_rank=6 min=4029 min_rank=4 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[556, 540, 514, 501, 519, 533, 525, 464, 0, 0, 0, 0, 0, 0, 0, 0, 504, 544, 533, 498, 464, 520, 540, 534, 20, 0, 0, 0, 0, 0, 0, 0, 511, 517, 511, 523, 526, 500, 489, 487, 0, 0, 0, 0, 0, 0, 0, 0, 529, 509, 515, 498, 541, 513, 435, 488, 0, 0, 0, 0, 0, 0, 0, 0, 478, 491, 523, 527, 491, 514, 497, 494, 15, 0, 0, 0, 0, 0, 0, 0, 506, 493, 480, 473, 514, 505, 532, 502, 8, 0, 0, 0, 0, 0, 0, 0, 523, 498, 514, 557, 494, 510, 488, 483, 45, 13, 0, 0, 0, 0, 0, 0, 521, 512, 528, 533, 500, 533, 491, 544, 29, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4152, 4157, 4064, 4028, 4030, 4013, 4125, 4199] avg=4096.0000 max=4199 max_rank=7 min=4013 min_rank=5 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[469, 514, 525, 522, 537, 529, 548, 495, 0, 0, 0, 0, 0, 0, 0, 0, 475, 523, 515, 524, 508, 526, 492, 477, 19, 0, 0, 0, 0, 0, 0, 0, 513, 499, 492, 535, 524, 514, 492, 521, 0, 0, 0, 0, 0, 0, 0, 0, 511, 506, 532, 512, 505, 471, 503, 485, 0, 0, 0, 0, 0, 0, 0, 0, 525, 516, 493, 474, 504, 507, 513, 534, 11, 0, 0, 0, 0, 0, 0, 0, 511, 509, 536, 547, 539, 510, 532, 513, 1, 0, 0, 0, 0, 0, 0, 0, 488, 460, 471, 513, 498, 522, 541, 501, 42, 9, 0, 0, 0, 0, 0, 0, 510, 511, 509, 524, 510, 558, 479, 507, 26, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4059, 4090, 4025, 4077, 4198, 4045, 4135] avg=4096.0000 max=4198 max_rank=5 min=4025 min_rank=3 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 3968, 4096, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 3968, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[527, 510, 543, 500, 489, 520, 500, 486, 503, 522, 529, 505, 501, 466, 517, 509, 557, 495, 474, 490, 541, 504, 489, 501, 539, 522, 497, 512, 510, 520, 541, 482, 502, 507, 540, 529, 531, 553, 495, 535, 522, 487, 495, 513, 518, 503, 496, 487, 515, 500, 511, 509, 509, 507, 479, 523, 518, 519, 468, 587, 550, 482, 550, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[520, 512, 492, 533, 535, 476, 520, 539, 533, 501, 533, 524, 489, 522, 467, 512, 534, 542, 494, 516, 494, 490, 490, 480, 523, 485, 518, 558, 514, 514, 532, 532, 509, 495, 523, 502, 552, 518, 495, 491, 526, 523, 523, 527, 495, 498, 510, 516, 533, 542, 487, 504, 510, 505, 518, 533, 480, 480, 484, 483, 513, 521, 524, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[539, 512, 543, 506, 504, 484, 500, 516, 498, 490, 506, 547, 496, 553, 533, 533, 467, 512, 513, 534, 503, 503, 503, 514, 486, 485, 494, 520, 552, 507, 501, 500, 483, 507, 466, 486, 549, 537, 539, 525, 528, 548, 518, 501, 522, 537, 479, 535, 469, 553, 521, 507, 518, 469, 534, 505, 497, 468, 534, 546, 502, 510, 509, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[488, 514, 556, 546, 500, 519, 535, 499, 536, 483, 485, 484, 528, 491, 521, 516, 536, 507, 513, 523, 480, 540, 495, 474, 546, 520, 540, 535, 516, 501, 535, 525, 501, 551, 526, 483, 529, 524, 529, 527, 489, 524, 516, 501, 524, 510, 475, 498, 535, 467, 528, 514, 496, 540, 493, 496, 507, 508, 519, 516, 465, 524, 471, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[508, 485, 482, 510, 504, 511, 537, 522, 461, 523, 525, 524, 485, 525, 518, 496, 524, 518, 507, 511, 515, 516, 517, 522, 557, 492, 522, 501, 491, 495, 476, 524, 530, 479, 513, 513, 545, 502, 572, 511, 525, 488, 545, 528, 527, 478, 492, 539, 482, 515, 523, 523, 512, 505, 541, 529, 541, 489, 498, 537, 478, 513, 498, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[530, 546, 517, 511, 470, 505, 545, 519, 483, 492, 492, 492, 538, 530, 505, 533, 511, 496, 498, 496, 574, 518, 510, 518, 507, 533, 486, 516, 532, 520, 504, 479, 541, 495, 512, 510, 516, 493, 502, 499, 533, 518, 522, 493, 527, 522, 520, 499, 536, 547, 554, 494, 483, 502, 479, 525, 530, 497, 498, 481, 502, 536, 475, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[513, 512, 523, 527, 510, 531, 520, 494, 498, 491, 507, 518, 544, 506, 496, 540, 531, 519, 514, 542, 510, 489, 498, 483, 499, 537, 498, 499, 500, 508, 510, 493, 490, 505, 525, 509, 495, 510, 511, 473, 546, 544, 544, 482, 518, 559, 491, 483, 536, 528, 513, 517, 523, 475, 539, 536, 487, 491, 493, 505, 514, 523, 505, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[502, 487, 528, 531, 524, 512, 540, 539, 532, 512, 518, 546, 497, 519, 474, 509, 517, 539, 505, 515, 497, 475, 535, 533, 494, 577, 492, 523, 520, 483, 478, 504, 550, 484, 501, 524, 503, 453, 539, 533, 524, 553, 520, 497, 495, 537, 527, 528, 538, 513, 499, 512, 511, 463, 539, 542, 460, 486, 474, 468, 493, 511, 530, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[527, 510, 543, 500, 489, 520, 478, 486, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 529, 505, 501, 466, 517, 509, 7, 5, 0, 0, 0, 0, 0, 0, 557, 495, 474, 490, 541, 504, 489, 501, 9, 4, 0, 0, 0, 0, 0, 0, 539, 522, 497, 507, 510, 520, 541, 482, 0, 0, 0, 0, 0, 0, 0, 0, 502, 507, 540, 529, 518, 553, 495, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 487, 495, 513, 518, 503, 496, 487, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 511, 509, 509, 507, 479, 512, 0, 0, 0, 0, 0, 0, 0, 0, 518, 519, 468, 587, 550, 482, 550, 527, 22, 22, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4064, 4064, 4118, 4179, 3999, 4042, 4249] avg=4096.0000 max=4249 max_rank=7 min=3999 min_rank=5 imbalance_ratio=1.037354 max_load_reduction=28519 max_load_reduction_pct=0.870331 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 512, 492, 533, 535, 476, 497, 539, 0, 0, 0, 0, 0, 0, 0, 0, 533, 501, 533, 524, 489, 522, 467, 512, 7, 5, 0, 0, 0, 0, 0, 0, 534, 542, 494, 516, 494, 490, 490, 480, 9, 4, 0, 0, 0, 0, 0, 0, 523, 485, 518, 553, 514, 514, 532, 532, 0, 0, 0, 0, 0, 0, 0, 0, 509, 495, 523, 502, 539, 518, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 504, 523, 523, 527, 495, 498, 510, 516, 0, 0, 0, 0, 0, 0, 0, 0, 533, 542, 487, 504, 510, 505, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 480, 480, 484, 483, 513, 521, 524, 524, 23, 22, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4093, 4053, 4171, 4072, 4096, 4121, 4058] avg=4096.0000 max=4171 max_rank=3 min=4053 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 485, 482, 510, 504, 511, 513, 522, 0, 0, 0, 0, 0, 0, 0, 0, 461, 523, 525, 524, 485, 525, 518, 496, 7, 5, 0, 0, 0, 0, 0, 0, 524, 518, 507, 511, 515, 516, 517, 522, 9, 4, 0, 0, 0, 0, 0, 0, 557, 492, 522, 496, 491, 495, 476, 524, 0, 0, 0, 0, 0, 0, 0, 0, 530, 479, 513, 513, 532, 502, 572, 511, 0, 0, 0, 0, 0, 0, 0, 0, 503, 488, 545, 528, 527, 478, 492, 539, 0, 0, 0, 0, 0, 0, 0, 0, 482, 515, 523, 523, 512, 505, 541, 518, 0, 0, 0, 0, 0, 0, 0, 0, 541, 489, 498, 537, 478, 513, 498, 493, 24, 22, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4035, 4069, 4143, 4053, 4152, 4100, 4119, 4097] avg=4096.0000 max=4152 max_rank=4 min=4035 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16468 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[530, 546, 517, 511, 470, 505, 521, 519, 0, 0, 0, 0, 0, 0, 0, 0, 483, 492, 492, 492, 538, 530, 505, 533, 7, 5, 0, 0, 0, 0, 0, 0, 511, 496, 498, 496, 574, 518, 510, 518, 8, 4, 0, 0, 0, 0, 0, 0, 507, 533, 486, 511, 532, 520, 504, 479, 0, 0, 0, 0, 0, 0, 0, 0, 541, 495, 512, 510, 504, 493, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 518, 522, 493, 527, 522, 520, 499, 0, 0, 0, 0, 0, 0, 0, 0, 536, 547, 554, 494, 483, 502, 479, 514, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 498, 481, 502, 536, 475, 521, 24, 22, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4077, 4133, 4072, 4056, 4112, 4109, 4090] avg=4096.0000 max=4133 max_rank=2 min=4056 min_rank=4 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16367 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[539, 512, 543, 506, 504, 484, 478, 516, 0, 0, 0, 0, 0, 0, 0, 0, 498, 490, 506, 547, 496, 553, 533, 533, 7, 5, 0, 0, 0, 0, 0, 0, 467, 512, 513, 534, 503, 503, 503, 514, 9, 4, 0, 0, 0, 0, 0, 0, 486, 485, 494, 515, 552, 507, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 483, 507, 466, 486, 536, 537, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 506, 548, 518, 501, 522, 537, 479, 535, 0, 0, 0, 0, 0, 0, 0, 0, 469, 553, 521, 507, 518, 469, 534, 494, 0, 0, 0, 0, 0, 0, 0, 0, 497, 468, 534, 546, 502, 510, 509, 512, 22, 22, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4168, 4062, 4040, 4079, 4146, 4065, 4126] avg=4096.0000 max=4168 max_rank=1 min=4040 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 512, 523, 527, 510, 531, 497, 494, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 507, 518, 544, 506, 496, 540, 7, 5, 0, 0, 0, 0, 0, 0, 531, 519, 514, 542, 510, 489, 498, 483, 8, 4, 0, 0, 0, 0, 0, 0, 499, 537, 498, 494, 500, 508, 510, 493, 0, 0, 0, 0, 0, 0, 0, 0, 490, 505, 525, 509, 483, 510, 511, 473, 0, 0, 0, 0, 0, 0, 0, 0, 523, 544, 544, 482, 518, 559, 491, 483, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 513, 517, 523, 475, 539, 525, 0, 0, 0, 0, 0, 0, 0, 0, 487, 491, 493, 505, 514, 523, 505, 538, 23, 23, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4112, 4098, 4039, 4006, 4144, 4156, 4106] avg=4096.0000 max=4156 max_rank=6 min=4006 min_rank=4 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 514, 556, 546, 500, 519, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 536, 483, 485, 484, 528, 491, 521, 516, 7, 5, 0, 0, 0, 0, 0, 0, 536, 507, 513, 523, 480, 540, 495, 474, 9, 4, 0, 0, 0, 0, 0, 0, 546, 520, 540, 530, 516, 501, 535, 525, 0, 0, 0, 0, 0, 0, 0, 0, 501, 551, 526, 483, 516, 524, 529, 527, 0, 0, 0, 0, 0, 0, 0, 0, 469, 524, 516, 501, 524, 510, 475, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 467, 528, 514, 496, 540, 493, 485, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 519, 516, 465, 524, 471, 495, 24, 20, 4, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4056, 4081, 4213, 4157, 4017, 4058, 4053] avg=4096.0000 max=4213 max_rank=3 min=4017 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[502, 487, 528, 531, 524, 512, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 518, 546, 497, 519, 474, 509, 13, 9, 0, 0, 0, 0, 0, 0, 517, 539, 505, 515, 497, 475, 535, 533, 12, 6, 0, 0, 0, 0, 0, 0, 494, 577, 492, 514, 520, 483, 478, 504, 0, 0, 0, 0, 0, 0, 0, 0, 550, 484, 501, 524, 483, 453, 539, 533, 0, 0, 0, 0, 0, 0, 0, 0, 497, 553, 520, 497, 495, 537, 527, 528, 0, 0, 0, 0, 0, 0, 0, 0, 538, 513, 499, 512, 511, 463, 539, 523, 0, 0, 0, 0, 0, 0, 0, 0, 460, 486, 474, 468, 493, 511, 530, 504, 28, 27, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4135, 4129, 4134, 4062, 4067, 4154, 4098, 3989] avg=4096.0000 max=4154 max_rank=5 min=3989 min_rank=7 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 3968, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 3968, 3968, 4224, 4096, 4224, 4096, 4224, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[516, 498, 517, 557, 503, 509, 472, 516, 510, 540, 502, 513, 486, 517, 510, 529, 518, 500, 508, 561, 531, 491, 523, 562, 514, 506, 511, 537, 493, 513, 503, 530, 495, 523, 565, 510, 498, 524, 499, 497, 529, 481, 495, 478, 491, 528, 527, 516, 516, 510, 510, 482, 487, 522, 518, 516, 504, 530, 508, 473, 493, 494, 545, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[510, 528, 535, 515, 531, 504, 528, 498, 516, 546, 538, 510, 536, 500, 505, 511, 514, 473, 557, 517, 508, 549, 513, 512, 531, 494, 533, 503, 481, 512, 492, 478, 547, 495, 484, 499, 488, 519, 484, 508, 535, 517, 510, 497, 532, 504, 523, 483, 519, 504, 505, 525, 528, 525, 485, 540, 481, 472, 512, 522, 519, 517, 507, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[510, 515, 510, 478, 556, 532, 536, 490, 529, 554, 520, 525, 554, 516, 516, 503, 495, 509, 547, 494, 502, 496, 525, 526, 515, 487, 509, 524, 515, 538, 536, 537, 499, 488, 521, 514, 510, 520, 515, 493, 494, 496, 529, 483, 506, 486, 481, 515, 501, 468, 527, 498, 534, 543, 533, 504, 513, 505, 504, 525, 481, 506, 480, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[514, 480, 489, 528, 491, 485, 530, 511, 499, 516, 546, 479, 494, 528, 511, 541, 491, 522, 482, 557, 524, 555, 507, 484, 481, 495, 507, 474, 525, 529, 539, 524, 471, 521, 518, 486, 518, 535, 556, 565, 515, 500, 502, 501, 517, 507, 517, 547, 506, 501, 536, 489, 511, 517, 484, 504, 508, 544, 529, 532, 496, 517, 469, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[526, 505, 520, 488, 481, 486, 532, 518, 510, 482, 493, 497, 548, 516, 525, 512, 498, 537, 546, 511, 507, 516, 533, 511, 499, 495, 496, 508, 497, 489, 488, 537, 534, 464, 500, 502, 479, 487, 500, 545, 543, 513, 503, 547, 467, 517, 510, 459, 540, 540, 515, 484, 525, 496, 556, 515, 510, 543, 537, 506, 548, 527, 521, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[476, 521, 495, 520, 533, 527, 529, 522, 505, 527, 524, 537, 494, 517, 480, 517, 520, 528, 541, 502, 524, 492, 520, 486, 480, 488, 524, 571, 534, 488, 506, 556, 534, 542, 525, 482, 523, 539, 489, 486, 529, 530, 476, 507, 533, 467, 489, 510, 519, 502, 489, 514, 510, 525, 425, 535, 531, 527, 493, 512, 490, 486, 537, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[484, 514, 501, 489, 511, 540, 509, 496, 519, 523, 508, 542, 519, 506, 504, 546, 516, 511, 512, 534, 511, 507, 515, 561, 522, 481, 497, 525, 479, 477, 495, 545, 470, 474, 527, 509, 506, 505, 524, 531, 516, 511, 487, 522, 531, 540, 548, 517, 476, 529, 465, 510, 534, 497, 540, 512, 523, 500, 493, 498, 502, 531, 525, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[473, 492, 517, 553, 520, 536, 537, 549, 540, 488, 541, 489, 510, 529, 530, 505, 506, 494, 498, 567, 479, 512, 515, 495, 521, 495, 525, 529, 500, 529, 532, 478, 518, 482, 502, 520, 460, 503, 498, 518, 511, 481, 499, 486, 515, 516, 527, 529, 580, 529, 530, 528, 515, 474, 481, 535, 513, 516, 495, 506, 540, 465, 502, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[516, 498, 517, 557, 503, 509, 470, 516, 0, 0, 0, 0, 0, 0, 0, 0, 510, 500, 502, 513, 486, 517, 510, 529, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 508, 524, 531, 491, 523, 562, 0, 0, 0, 0, 0, 0, 0, 0, 514, 506, 511, 537, 493, 513, 503, 530, 15, 0, 0, 0, 0, 0, 0, 0, 495, 523, 565, 510, 498, 524, 499, 497, 29, 0, 0, 0, 0, 0, 0, 0, 529, 481, 495, 478, 491, 528, 527, 516, 13, 11, 0, 0, 0, 0, 0, 0, 516, 510, 510, 482, 487, 522, 518, 516, 0, 0, 0, 0, 0, 0, 0, 0, 504, 530, 508, 473, 493, 494, 545, 508, 9, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4086, 4067, 4157, 4122, 4140, 4069, 4061, 4066] avg=4096.0000 max=4157 max_rank=2 min=4061 min_rank=6 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 528, 535, 515, 531, 504, 525, 498, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 538, 510, 536, 500, 505, 511, 0, 0, 0, 0, 0, 0, 0, 0, 514, 473, 557, 482, 508, 549, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 533, 503, 481, 512, 492, 478, 14, 0, 0, 0, 0, 0, 0, 0, 547, 495, 484, 499, 488, 519, 484, 508, 29, 0, 0, 0, 0, 0, 0, 0, 535, 517, 510, 497, 532, 504, 523, 483, 12, 11, 0, 0, 0, 0, 0, 0, 519, 504, 505, 525, 528, 525, 485, 540, 0, 0, 0, 0, 0, 0, 0, 0, 481, 472, 512, 522, 519, 517, 507, 504, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4122, 4108, 4038, 4053, 4124, 4131, 4046] avg=4096.0000 max=4146 max_rank=0 min=4038 min_rank=3 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 480, 489, 528, 491, 485, 527, 511, 0, 0, 0, 0, 0, 0, 0, 0, 499, 478, 546, 479, 494, 528, 511, 541, 0, 0, 0, 0, 0, 0, 0, 0, 491, 522, 482, 520, 524, 555, 507, 484, 0, 0, 0, 0, 0, 0, 0, 0, 481, 495, 507, 474, 525, 529, 539, 524, 15, 0, 0, 0, 0, 0, 0, 0, 471, 521, 518, 486, 518, 535, 556, 565, 27, 0, 0, 0, 0, 0, 0, 0, 515, 500, 502, 501, 517, 507, 517, 547, 13, 11, 0, 0, 0, 0, 0, 0, 506, 501, 536, 489, 511, 517, 484, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 544, 529, 532, 496, 517, 469, 511, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4076, 4085, 4089, 4197, 4130, 4048, 4118] avg=4096.0000 max=4197 max_rank=4 min=4025 min_rank=0 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 515, 510, 478, 556, 532, 533, 490, 0, 0, 0, 0, 0, 0, 0, 0, 529, 514, 520, 525, 554, 516, 516, 503, 0, 0, 0, 0, 0, 0, 0, 0, 495, 509, 547, 462, 502, 496, 525, 526, 0, 0, 0, 0, 0, 0, 0, 0, 515, 487, 509, 524, 515, 538, 536, 537, 13, 0, 0, 0, 0, 0, 0, 0, 499, 488, 521, 514, 510, 520, 515, 493, 29, 0, 0, 0, 0, 0, 0, 0, 494, 496, 529, 483, 506, 486, 481, 515, 11, 11, 0, 0, 0, 0, 0, 0, 501, 468, 527, 498, 534, 543, 533, 504, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 504, 525, 481, 506, 480, 497, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4177, 4062, 4174, 4089, 4012, 4108, 4022] avg=4096.0000 max=4177 max_rank=1 min=4012 min_rank=5 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16231 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[526, 505, 520, 488, 481, 486, 529, 518, 0, 0, 0, 0, 0, 0, 0, 0, 510, 447, 493, 497, 548, 516, 525, 512, 0, 0, 0, 0, 0, 0, 0, 0, 498, 537, 546, 478, 507, 516, 533, 511, 0, 0, 0, 0, 0, 0, 0, 0, 499, 495, 496, 508, 497, 489, 488, 537, 13, 0, 0, 0, 0, 0, 0, 0, 534, 464, 500, 502, 479, 487, 500, 545, 25, 0, 0, 0, 0, 0, 0, 0, 543, 513, 503, 547, 467, 517, 510, 459, 12, 10, 0, 0, 0, 0, 0, 0, 540, 540, 515, 484, 525, 496, 556, 515, 0, 0, 0, 0, 0, 0, 0, 0, 510, 543, 537, 506, 548, 527, 521, 528, 8, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4053, 4048, 4126, 4022, 4036, 4081, 4171, 4231] avg=4096.0000 max=4231 max_rank=7 min=4022 min_rank=3 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16519 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[484, 514, 501, 489, 511, 540, 507, 496, 0, 0, 0, 0, 0, 0, 0, 0, 519, 484, 508, 542, 519, 506, 504, 546, 0, 0, 0, 0, 0, 0, 0, 0, 516, 511, 512, 499, 511, 507, 515, 561, 0, 0, 0, 0, 0, 0, 0, 0, 522, 481, 497, 525, 479, 477, 495, 545, 14, 0, 0, 0, 0, 0, 0, 0, 470, 474, 527, 509, 506, 505, 524, 531, 28, 0, 0, 0, 0, 0, 0, 0, 516, 511, 487, 522, 531, 540, 548, 517, 12, 11, 0, 0, 0, 0, 0, 0, 476, 529, 465, 510, 534, 497, 540, 512, 0, 0, 0, 0, 0, 0, 0, 0, 523, 500, 493, 498, 502, 531, 525, 516, 9, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4128, 4132, 4035, 4074, 4195, 4063, 4099] avg=4096.0000 max=4195 max_rank=5 min=4035 min_rank=3 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[476, 521, 495, 520, 533, 527, 524, 522, 0, 0, 0, 0, 0, 0, 0, 0, 505, 480, 524, 537, 494, 517, 480, 517, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 541, 458, 524, 492, 520, 486, 0, 0, 0, 0, 0, 0, 0, 0, 480, 488, 524, 571, 534, 488, 506, 556, 17, 0, 0, 0, 0, 0, 0, 0, 534, 542, 525, 482, 523, 539, 489, 486, 32, 0, 0, 0, 0, 0, 0, 0, 529, 530, 476, 507, 533, 467, 489, 510, 14, 15, 0, 0, 0, 0, 0, 0, 519, 502, 489, 514, 510, 525, 425, 530, 0, 0, 0, 0, 0, 0, 0, 0, 531, 527, 493, 512, 490, 486, 537, 528, 13, 5, 5, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4054, 4069, 4164, 4152, 4070, 4014, 4127] avg=4096.0000 max=4164 max_rank=3 min=4014 min_rank=6 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[473, 492, 517, 553, 520, 536, 534, 549, 0, 0, 0, 0, 0, 0, 0, 0, 540, 452, 541, 489, 510, 529, 530, 505, 0, 0, 0, 0, 0, 0, 0, 0, 506, 494, 498, 530, 479, 512, 515, 495, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 525, 529, 500, 529, 532, 478, 15, 0, 0, 0, 0, 0, 0, 0, 518, 482, 502, 520, 460, 503, 498, 518, 26, 0, 0, 0, 0, 0, 0, 0, 511, 481, 499, 486, 515, 516, 527, 529, 13, 10, 0, 0, 0, 0, 0, 0, 580, 529, 530, 528, 515, 474, 481, 535, 0, 0, 0, 0, 0, 0, 0, 0, 513, 516, 495, 506, 540, 465, 502, 510, 9, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4096, 4029, 4124, 4027, 4087, 4172, 4059] avg=4096.0000 max=4174 max_rank=0 min=4027 min_rank=4 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4224, 4224, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 14:57:43] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 284.0 | throughput per GPU (TFLOP/s/GPU): 434.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.766685E+00 | loss scale: 1.0 | grad norm: 4.121 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[509, 519, 513, 545, 556, 496, 520, 471, 550, 526, 506, 501, 546, 508, 522, 524, 517, 531, 526, 519, 553, 530, 511, 481, 556, 505, 479, 494, 536, 480, 519, 490, 477, 457, 507, 501, 521, 535, 498, 515, 477, 528, 508, 482, 491, 492, 523, 534, 554, 486, 521, 522, 504, 485, 526, 548, 494, 497, 456, 499, 521, 543, 503, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[552, 484, 512, 512, 531, 489, 503, 524, 494, 542, 500, 534, 527, 516, 522, 523, 548, 528, 488, 504, 533, 503, 504, 482, 480, 514, 483, 529, 527, 490, 524, 541, 542, 501, 482, 511, 539, 512, 464, 499, 529, 474, 537, 460, 475, 485, 490, 492, 525, 509, 542, 517, 511, 525, 481, 539, 522, 522, 537, 528, 546, 513, 496, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[533, 534, 511, 526, 514, 513, 494, 488, 537, 528, 508, 499, 512, 544, 542, 550, 476, 524, 482, 540, 522, 499, 504, 524, 473, 524, 516, 498, 503, 534, 520, 522, 496, 535, 514, 539, 488, 495, 486, 487, 478, 543, 523, 517, 501, 494, 521, 507, 550, 523, 540, 521, 496, 478, 529, 505, 501, 504, 507, 479, 502, 500, 494, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[475, 507, 477, 543, 486, 510, 532, 516, 531, 491, 488, 528, 476, 534, 566, 504, 485, 518, 509, 485, 546, 507, 527, 538, 509, 544, 515, 536, 466, 517, 540, 529, 503, 500, 555, 531, 505, 517, 527, 501, 507, 498, 519, 527, 504, 481, 520, 493, 479, 510, 503, 523, 526, 506, 493, 526, 524, 541, 493, 516, 510, 490, 510, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[486, 515, 511, 519, 516, 503, 539, 501, 543, 519, 514, 529, 493, 548, 507, 502, 501, 526, 505, 533, 543, 536, 517, 539, 501, 554, 531, 502, 490, 533, 487, 497, 524, 506, 523, 532, 494, 474, 495, 508, 528, 518, 470, 500, 490, 448, 525, 519, 531, 493, 547, 494, 490, 520, 517, 509, 543, 525, 504, 529, 518, 482, 490, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[574, 522, 541, 523, 520, 509, 483, 479, 535, 475, 507, 511, 519, 523, 547, 521, 518, 549, 509, 495, 524, 541, 544, 498, 496, 503, 535, 542, 521, 528, 487, 488, 495, 512, 466, 528, 530, 488, 522, 547, 479, 491, 514, 508, 496, 546, 490, 512, 502, 515, 493, 473, 520, 512, 509, 502, 553, 511, 491, 488, 504, 498, 487, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[512, 527, 530, 524, 521, 497, 527, 514, 491, 536, 545, 514, 546, 485, 478, 539, 531, 496, 556, 480, 516, 514, 476, 502, 521, 496, 556, 514, 523, 501, 561, 521, 520, 461, 507, 484, 538, 519, 531, 483, 498, 512, 519, 506, 470, 526, 476, 494, 512, 496, 512, 487, 517, 531, 489, 502, 503, 512, 493, 500, 579, 530, 523, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[517, 505, 518, 478, 512, 500, 566, 500, 503, 507, 572, 510, 514, 480, 501, 505, 484, 527, 536, 523, 510, 508, 515, 524, 539, 534, 515, 507, 535, 511, 501, 526, 521, 543, 530, 495, 541, 455, 498, 511, 515, 487, 497, 506, 535, 484, 489, 519, 492, 524, 519, 474, 532, 512, 513, 510, 527, 501, 510, 497, 495, 541, 494, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[533, 534, 511, 512, 514, 513, 494, 488, 0, 0, 0, 0, 0, 0, 0, 0, 537, 528, 508, 499, 512, 544, 487, 550, 0, 0, 0, 0, 0, 0, 0, 0, 476, 524, 482, 540, 492, 499, 504, 524, 0, 0, 0, 0, 0, 0, 0, 0, 473, 502, 516, 498, 503, 534, 520, 522, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 514, 539, 488, 495, 486, 487, 19, 11, 0, 0, 0, 0, 0, 0, 478, 543, 523, 517, 501, 494, 521, 507, 55, 19, 0, 0, 0, 0, 0, 0, 550, 523, 540, 521, 496, 478, 529, 505, 2, 0, 0, 0, 0, 0, 0, 0, 501, 504, 507, 479, 502, 500, 494, 521, 12, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4165, 4041, 4068, 4070, 4158, 4144, 4023] avg=4096.0000 max=4165 max_rank=1 min=4023 min_rank=7 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 519, 513, 530, 556, 496, 520, 471, 0, 0, 0, 0, 0, 0, 0, 0, 550, 526, 506, 501, 546, 508, 469, 524, 0, 0, 0, 0, 0, 0, 0, 0, 517, 531, 526, 519, 522, 530, 511, 481, 0, 0, 0, 0, 0, 0, 0, 0, 556, 484, 479, 494, 536, 480, 519, 490, 0, 0, 0, 0, 0, 0, 0, 0, 477, 457, 507, 501, 521, 535, 498, 515, 18, 11, 0, 0, 0, 0, 0, 0, 477, 528, 508, 482, 491, 492, 523, 534, 53, 20, 0, 0, 0, 0, 0, 0, 554, 486, 521, 522, 504, 485, 526, 548, 2, 0, 0, 0, 0, 0, 0, 0, 494, 497, 456, 499, 521, 543, 503, 524, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4130, 4137, 4038, 4040, 4108, 4148, 4053] avg=4096.0000 max=4148 max_rank=6 min=4038 min_rank=3 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[486, 515, 511, 506, 516, 503, 539, 501, 0, 0, 0, 0, 0, 0, 0, 0, 543, 519, 514, 529, 493, 548, 455, 502, 0, 0, 0, 0, 0, 0, 0, 0, 501, 526, 505, 533, 512, 536, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 501, 531, 531, 502, 490, 533, 487, 497, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 523, 532, 494, 474, 495, 508, 20, 11, 0, 0, 0, 0, 0, 0, 528, 518, 470, 500, 490, 448, 525, 519, 52, 20, 0, 0, 0, 0, 0, 0, 531, 493, 547, 494, 490, 520, 517, 509, 1, 0, 0, 0, 0, 0, 0, 0, 543, 525, 504, 529, 518, 482, 490, 482, 12, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4103, 4169, 4072, 4087, 4070, 4102, 4088] avg=4096.0000 max=4169 max_rank=2 min=4070 min_rank=5 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[512, 527, 530, 510, 521, 497, 527, 514, 0, 0, 0, 0, 0, 0, 0, 0, 491, 536, 545, 514, 546, 485, 429, 539, 0, 0, 0, 0, 0, 0, 0, 0, 531, 496, 556, 480, 486, 514, 476, 502, 0, 0, 0, 0, 0, 0, 0, 0, 521, 476, 556, 514, 523, 501, 561, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 461, 507, 484, 538, 519, 531, 483, 18, 11, 0, 0, 0, 0, 0, 0, 498, 512, 519, 506, 470, 526, 476, 494, 49, 19, 0, 0, 0, 0, 0, 0, 512, 496, 512, 487, 517, 531, 489, 502, 2, 0, 0, 0, 0, 0, 0, 0, 503, 512, 493, 500, 579, 530, 523, 488, 12, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 4085, 4041, 4173, 4072, 4069, 4048, 4142] avg=4096.0000 max=4173 max_rank=3 min=4041 min_rank=2 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[475, 507, 477, 528, 486, 510, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 531, 491, 488, 528, 476, 534, 508, 504, 0, 0, 0, 0, 0, 0, 0, 0, 485, 518, 509, 485, 515, 507, 527, 538, 0, 0, 0, 0, 0, 0, 0, 0, 509, 521, 515, 536, 466, 517, 540, 529, 0, 0, 0, 0, 0, 0, 0, 0, 503, 500, 555, 531, 505, 517, 527, 501, 20, 11, 0, 0, 0, 0, 0, 0, 507, 498, 519, 527, 504, 481, 520, 493, 58, 20, 0, 0, 0, 0, 0, 0, 479, 510, 503, 523, 526, 506, 493, 526, 2, 0, 0, 0, 0, 0, 0, 0, 524, 541, 493, 516, 510, 490, 510, 495, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4031, 4060, 4084, 4133, 4170, 4127, 4068, 4095] avg=4096.0000 max=4170 max_rank=4 min=4031 min_rank=0 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16460 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[574, 522, 541, 509, 520, 509, 483, 479, 0, 0, 0, 0, 0, 0, 0, 0, 535, 475, 507, 511, 519, 523, 491, 521, 0, 0, 0, 0, 0, 0, 0, 0, 518, 549, 509, 495, 494, 541, 544, 498, 0, 0, 0, 0, 0, 0, 0, 0, 496, 482, 535, 542, 521, 528, 487, 488, 0, 0, 0, 0, 0, 0, 0, 0, 495, 512, 466, 528, 530, 488, 522, 547, 18, 11, 0, 0, 0, 0, 0, 0, 479, 491, 514, 508, 496, 546, 490, 512, 56, 19, 0, 0, 0, 0, 0, 0, 502, 515, 493, 473, 520, 512, 509, 502, 2, 0, 0, 0, 0, 0, 0, 0, 553, 511, 491, 488, 504, 498, 487, 519, 12, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4082, 4148, 4079, 4117, 4111, 4028, 4066] avg=4096.0000 max=4148 max_rank=2 min=4028 min_rank=6 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16322 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[552, 484, 512, 493, 531, 489, 503, 524, 0, 0, 0, 0, 0, 0, 0, 0, 494, 542, 500, 534, 527, 516, 466, 523, 0, 0, 0, 0, 0, 0, 0, 0, 548, 528, 488, 504, 493, 503, 504, 482, 0, 0, 0, 0, 0, 0, 0, 0, 480, 485, 483, 529, 527, 490, 524, 541, 0, 0, 0, 0, 0, 0, 0, 0, 542, 501, 482, 511, 539, 512, 464, 499, 24, 15, 0, 0, 0, 0, 0, 0, 529, 474, 537, 460, 475, 485, 490, 492, 56, 25, 0, 0, 0, 0, 0, 0, 525, 509, 542, 517, 511, 525, 481, 539, 4, 0, 0, 0, 0, 0, 0, 0, 522, 522, 537, 528, 546, 513, 496, 520, 15, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4102, 4050, 4059, 4089, 4023, 4153, 4204] avg=4096.0000 max=4204 max_rank=7 min=4023 min_rank=5 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[517, 505, 518, 466, 512, 500, 566, 500, 0, 0, 0, 0, 0, 0, 0, 0, 503, 507, 572, 510, 514, 480, 450, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 527, 536, 523, 481, 508, 515, 524, 0, 0, 0, 0, 0, 0, 0, 0, 539, 512, 515, 507, 535, 511, 501, 526, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 530, 495, 541, 455, 498, 511, 19, 10, 0, 0, 0, 0, 0, 0, 515, 487, 497, 506, 535, 484, 489, 519, 51, 19, 0, 0, 0, 0, 0, 0, 492, 524, 519, 474, 532, 512, 513, 510, 1, 0, 0, 0, 0, 0, 0, 0, 527, 501, 510, 497, 495, 541, 494, 518, 11, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4041, 4098, 4146, 4123, 4102, 4077, 4097] avg=4096.0000 max=4146 max_rank=3 min=4041 min_rank=1 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4224, 3840, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 3968, 4096, 4096, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[570, 505, 483, 523, 506, 519, 499, 529, 518, 518, 525, 535, 496, 566, 550, 499, 483, 484, 501, 542, 515, 515, 520, 511, 516, 496, 496, 503, 530, 498, 526, 490, 491, 511, 483, 520, 501, 525, 533, 515, 479, 515, 508, 492, 556, 530, 530, 536, 506, 517, 520, 505, 521, 502, 529, 508, 505, 503, 459, 524, 464, 510, 504, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[534, 468, 507, 526, 517, 492, 564, 507, 527, 553, 537, 538, 519, 539, 457, 539, 527, 475, 486, 526, 471, 533, 487, 516, 499, 509, 484, 502, 475, 548, 541, 504, 547, 469, 495, 514, 507, 519, 479, 496, 539, 543, 475, 482, 512, 527, 497, 554, 494, 519, 519, 516, 517, 522, 513, 557, 511, 512, 507, 520, 511, 507, 464, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[518, 537, 543, 502, 505, 507, 499, 539, 535, 504, 515, 512, 478, 520, 526, 519, 500, 529, 497, 497, 556, 492, 496, 525, 500, 485, 507, 485, 480, 517, 543, 479, 524, 519, 488, 516, 471, 542, 512, 526, 478, 497, 534, 488, 489, 533, 489, 473, 518, 510, 528, 532, 533, 502, 523, 493, 513, 562, 517, 534, 487, 539, 504, 547] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[542, 513, 534, 500, 504, 499, 541, 512, 506, 490, 509, 548, 511, 506, 515, 511, 512, 504, 539, 550, 509, 544, 534, 498, 488, 518, 497, 505, 522, 525, 528, 515, 519, 529, 538, 482, 468, 546, 495, 518, 488, 480, 490, 501, 498, 515, 537, 508, 524, 513, 516, 522, 465, 533, 481, 494, 566, 485, 524, 518, 479, 489, 491, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[504, 534, 498, 542, 499, 516, 554, 523, 482, 487, 506, 534, 507, 492, 499, 486, 515, 542, 499, 519, 545, 480, 541, 516, 506, 520, 550, 527, 489, 494, 500, 500, 517, 497, 502, 504, 497, 526, 493, 530, 550, 542, 498, 534, 511, 537, 497, 480, 504, 495, 460, 451, 522, 508, 496, 494, 523, 510, 496, 541, 549, 525, 529, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[530, 516, 531, 509, 504, 486, 536, 498, 508, 501, 504, 508, 493, 538, 513, 500, 541, 551, 509, 502, 492, 517, 538, 504, 507, 528, 499, 533, 511, 468, 505, 503, 525, 517, 527, 507, 491, 523, 512, 520, 512, 494, 494, 496, 526, 470, 509, 479, 498, 555, 540, 542, 507, 489, 465, 540, 498, 499, 490, 529, 551, 540, 499, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[514, 498, 524, 505, 471, 533, 517, 498, 487, 496, 508, 516, 495, 533, 504, 540, 531, 480, 530, 518, 511, 485, 483, 526, 522, 508, 532, 497, 499, 519, 528, 528, 519, 494, 509, 511, 513, 487, 542, 524, 536, 472, 511, 518, 515, 512, 520, 532, 542, 474, 526, 515, 517, 509, 480, 515, 500, 522, 539, 494, 529, 525, 506, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[495, 505, 503, 523, 537, 528, 510, 532, 524, 499, 511, 520, 507, 500, 522, 514, 513, 529, 509, 565, 540, 510, 505, 500, 496, 507, 519, 525, 497, 534, 506, 508, 467, 506, 538, 502, 498, 472, 496, 520, 493, 497, 538, 503, 482, 497, 525, 500, 513, 488, 483, 506, 506, 525, 492, 522, 529, 493, 547, 527, 513, 536, 536, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[534, 468, 507, 526, 517, 492, 532, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 553, 537, 527, 519, 539, 457, 539, 0, 0, 0, 0, 0, 0, 0, 0, 527, 475, 486, 508, 471, 533, 487, 516, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 484, 502, 475, 548, 541, 504, 19, 4, 0, 0, 0, 0, 0, 0, 547, 469, 495, 514, 507, 519, 479, 496, 12, 10, 0, 0, 0, 0, 0, 0, 539, 543, 475, 482, 512, 527, 497, 554, 28, 0, 0, 0, 0, 0, 0, 0, 494, 519, 519, 516, 517, 522, 513, 557, 11, 6, 0, 0, 0, 0, 0, 0, 511, 512, 507, 520, 511, 507, 464, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4198, 4003, 4085, 4048, 4157, 4174, 4020] avg=4096.0000 max=4198 max_rank=1 min=4003 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 537, 543, 502, 505, 507, 470, 539, 0, 0, 0, 0, 0, 0, 0, 0, 535, 504, 515, 502, 478, 520, 526, 519, 0, 0, 0, 0, 0, 0, 0, 0, 500, 529, 497, 480, 556, 492, 496, 525, 0, 0, 0, 0, 0, 0, 0, 0, 500, 485, 507, 485, 480, 517, 543, 479, 20, 4, 0, 0, 0, 0, 0, 0, 524, 519, 488, 516, 471, 542, 512, 526, 12, 10, 0, 0, 0, 0, 0, 0, 478, 497, 534, 488, 489, 533, 489, 473, 25, 0, 0, 0, 0, 0, 0, 0, 518, 510, 528, 532, 533, 502, 523, 493, 10, 5, 0, 0, 0, 0, 0, 0, 513, 562, 517, 534, 487, 539, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4099, 4075, 4020, 4120, 4006, 4154, 4173] avg=4096.0000 max=4173 max_rank=7 min=4006 min_rank=5 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[570, 505, 483, 523, 506, 519, 470, 529, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 525, 524, 496, 566, 550, 499, 0, 0, 0, 0, 0, 0, 0, 0, 483, 484, 501, 523, 515, 515, 520, 511, 0, 0, 0, 0, 0, 0, 0, 0, 516, 496, 496, 503, 530, 498, 526, 490, 18, 4, 0, 0, 0, 0, 0, 0, 491, 511, 483, 520, 501, 525, 533, 515, 13, 9, 0, 0, 0, 0, 0, 0, 479, 515, 508, 492, 556, 530, 530, 536, 25, 0, 0, 0, 0, 0, 0, 0, 506, 517, 520, 505, 521, 502, 529, 508, 11, 6, 0, 0, 0, 0, 0, 0, 505, 503, 459, 524, 464, 510, 504, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4196, 4052, 4077, 4101, 4171, 4125, 3941] avg=4096.0000 max=4196 max_rank=1 min=3941 min_rank=7 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[530, 516, 531, 509, 504, 486, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 504, 498, 493, 538, 513, 500, 0, 0, 0, 0, 0, 0, 0, 0, 541, 551, 509, 485, 492, 517, 538, 504, 0, 0, 0, 0, 0, 0, 0, 0, 507, 528, 499, 533, 511, 468, 505, 503, 20, 4, 0, 0, 0, 0, 0, 0, 525, 517, 527, 507, 491, 523, 512, 520, 12, 10, 0, 0, 0, 0, 0, 0, 512, 494, 494, 496, 526, 470, 509, 479, 27, 0, 0, 0, 0, 0, 0, 0, 498, 555, 540, 542, 507, 489, 465, 540, 10, 5, 0, 0, 0, 0, 0, 0, 498, 499, 490, 529, 551, 540, 499, 511, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4055, 4137, 4078, 4144, 4007, 4151, 4117] avg=4096.0000 max=4151 max_rank=6 min=4007 min_rank=5 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[504, 534, 498, 542, 499, 516, 522, 523, 0, 0, 0, 0, 0, 0, 0, 0, 482, 487, 506, 523, 507, 492, 499, 486, 0, 0, 0, 0, 0, 0, 0, 0, 515, 542, 499, 501, 545, 480, 541, 516, 0, 0, 0, 0, 0, 0, 0, 0, 506, 520, 550, 527, 489, 494, 500, 500, 20, 4, 0, 0, 0, 0, 0, 0, 517, 497, 502, 504, 497, 526, 493, 530, 12, 10, 0, 0, 0, 0, 0, 0, 550, 542, 498, 534, 511, 537, 497, 480, 28, 0, 0, 0, 0, 0, 0, 0, 504, 495, 460, 451, 522, 508, 496, 494, 11, 6, 0, 0, 0, 0, 0, 0, 523, 510, 496, 541, 549, 525, 529, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4138, 3982, 4139, 4110, 4088, 4177, 3947, 4187] avg=4096.0000 max=4187 max_rank=7 min=3947 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[542, 513, 534, 500, 504, 499, 510, 512, 0, 0, 0, 0, 0, 0, 0, 0, 506, 490, 509, 537, 511, 506, 515, 511, 0, 0, 0, 0, 0, 0, 0, 0, 512, 504, 539, 531, 509, 544, 534, 498, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 497, 505, 522, 525, 528, 515, 19, 4, 0, 0, 0, 0, 0, 0, 519, 529, 538, 482, 468, 546, 495, 518, 13, 10, 0, 0, 0, 0, 0, 0, 488, 480, 490, 501, 498, 515, 537, 508, 27, 0, 0, 0, 0, 0, 0, 0, 524, 513, 516, 522, 465, 533, 481, 494, 11, 6, 0, 0, 0, 0, 0, 0, 566, 485, 524, 518, 479, 489, 491, 498, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4085, 4171, 4121, 4118, 4044, 4065, 4050] avg=4096.0000 max=4171 max_rank=2 min=4044 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 498, 524, 505, 471, 533, 481, 498, 0, 0, 0, 0, 0, 0, 0, 0, 487, 496, 508, 503, 495, 533, 504, 540, 0, 0, 0, 0, 0, 0, 0, 0, 531, 480, 530, 493, 511, 485, 483, 526, 0, 0, 0, 0, 0, 0, 0, 0, 522, 508, 532, 497, 499, 519, 528, 528, 23, 6, 0, 0, 0, 0, 0, 0, 519, 494, 509, 511, 513, 487, 542, 524, 15, 13, 0, 0, 0, 0, 0, 0, 536, 472, 511, 518, 515, 512, 520, 532, 30, 0, 0, 0, 0, 0, 0, 0, 542, 474, 526, 515, 517, 509, 480, 515, 13, 10, 0, 0, 0, 0, 0, 0, 500, 522, 539, 494, 529, 525, 506, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4066, 4039, 4162, 4127, 4146, 4101, 4103] avg=4096.0000 max=4162 max_rank=3 min=4024 min_rank=0 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16477 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[495, 505, 503, 523, 537, 528, 481, 532, 0, 0, 0, 0, 0, 0, 0, 0, 524, 499, 511, 510, 507, 500, 522, 514, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 509, 546, 540, 510, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 496, 507, 519, 525, 497, 534, 506, 508, 19, 4, 0, 0, 0, 0, 0, 0, 467, 506, 538, 502, 498, 472, 496, 520, 13, 10, 0, 0, 0, 0, 0, 0, 493, 497, 538, 503, 482, 497, 525, 500, 25, 0, 0, 0, 0, 0, 0, 0, 513, 488, 483, 506, 506, 525, 492, 522, 10, 6, 0, 0, 0, 0, 0, 0, 529, 493, 547, 527, 513, 536, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4104, 4087, 4152, 4115, 4022, 4060, 4051, 4177] avg=4096.0000 max=4177 max_rank=7 min=4022 min_rank=4 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16310 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 3968, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[469, 511, 542, 524, 461, 525, 518, 523, 521, 545, 539, 488, 503, 522, 510, 518, 488, 471, 545, 495, 491, 528, 525, 514, 490, 508, 466, 489, 487, 501, 503, 509, 509, 496, 497, 538, 498, 528, 532, 521, 493, 503, 542, 525, 507, 480, 547, 481, 541, 550, 528, 517, 523, 530, 533, 497, 515, 540, 527, 523, 518, 520, 476, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[529, 477, 482, 529, 522, 503, 517, 535, 518, 556, 493, 508, 511, 528, 508, 502, 543, 506, 523, 489, 516, 501, 515, 513, 503, 521, 471, 509, 491, 504, 507, 526, 526, 505, 474, 543, 495, 501, 500, 569, 506, 541, 507, 543, 518, 481, 522, 527, 491, 534, 508, 510, 497, 505, 542, 503, 533, 510, 520, 460, 514, 532, 505, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 535, 487, 505, 515, 476, 519, 543, 509, 527, 544, 490, 505, 502, 493, 501, 536, 528, 493, 494, 509, 542, 516, 502, 498, 502, 503, 533, 494, 519, 489, 537, 521, 499, 522, 541, 518, 516, 470, 491, 546, 531, 547, 551, 488, 515, 486, 514, 513, 496, 545, 520, 487, 545, 493, 518, 499, 499, 511, 451, 524, 528, 517, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[531, 512, 508, 504, 525, 541, 504, 510, 493, 550, 484, 452, 496, 531, 518, 520, 518, 507, 545, 499, 511, 457, 499, 503, 517, 505, 508, 528, 551, 529, 516, 556, 543, 501, 507, 495, 514, 479, 486, 506, 512, 534, 507, 517, 536, 520, 514, 502, 509, 519, 452, 504, 530, 502, 518, 552, 520, 501, 494, 512, 497, 530, 506, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[522, 532, 494, 515, 473, 555, 527, 497, 539, 526, 501, 514, 506, 553, 495, 537, 493, 539, 461, 505, 499, 480, 510, 518, 547, 462, 527, 532, 494, 492, 573, 524, 487, 489, 519, 497, 510, 538, 511, 509, 481, 474, 535, 457, 496, 522, 515, 514, 486, 528, 519, 522, 510, 539, 515, 534, 532, 492, 509, 529, 533, 527, 502, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[519, 526, 528, 531, 508, 501, 506, 501, 502, 529, 531, 494, 496, 500, 519, 526, 499, 519, 509, 474, 508, 514, 535, 532, 542, 541, 544, 522, 476, 506, 532, 529, 476, 522, 515, 506, 493, 526, 492, 516, 521, 498, 520, 502, 525, 571, 516, 490, 509, 511, 480, 508, 464, 503, 480, 527, 513, 531, 485, 498, 538, 525, 508, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[476, 552, 552, 549, 504, 521, 524, 485, 502, 538, 502, 504, 510, 522, 526, 529, 503, 559, 486, 513, 542, 516, 491, 512, 495, 515, 480, 546, 506, 520, 520, 515, 504, 507, 500, 527, 493, 515, 484, 474, 539, 550, 490, 547, 522, 469, 521, 499, 492, 525, 505, 530, 484, 505, 535, 526, 499, 479, 499, 533, 483, 489, 499, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[521, 516, 503, 519, 513, 487, 487, 510, 555, 521, 540, 521, 543, 508, 504, 529, 552, 534, 523, 556, 489, 472, 529, 502, 510, 497, 507, 531, 512, 494, 484, 502, 482, 543, 525, 481, 542, 542, 497, 483, 480, 510, 535, 514, 495, 512, 549, 497, 511, 512, 529, 511, 529, 495, 538, 471, 502, 514, 477, 518, 536, 494, 489, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[469, 511, 542, 514, 461, 525, 518, 523, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 539, 488, 503, 522, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 488, 471, 545, 495, 491, 528, 525, 514, 7, 2, 0, 0, 0, 0, 0, 0, 490, 508, 466, 489, 487, 501, 503, 509, 2, 0, 0, 0, 0, 0, 0, 0, 509, 496, 497, 538, 498, 528, 532, 521, 18, 8, 2, 0, 0, 0, 0, 0, 493, 503, 524, 525, 507, 480, 547, 481, 0, 0, 0, 0, 0, 0, 0, 0, 541, 541, 528, 517, 523, 530, 533, 497, 0, 0, 0, 0, 0, 0, 0, 0, 515, 540, 527, 523, 518, 520, 476, 504, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4116, 4066, 3955, 4147, 4060, 4210, 4151] avg=4096.0000 max=4210 max_rank=6 min=3955 min_rank=3 imbalance_ratio=1.027832 max_load_reduction=28558 max_load_reduction_pct=0.871521 moved_to_echo=16568 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[531, 512, 508, 496, 525, 541, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 493, 520, 484, 452, 496, 531, 518, 520, 0, 0, 0, 0, 0, 0, 0, 0, 518, 507, 545, 499, 511, 457, 499, 503, 6, 1, 0, 0, 0, 0, 0, 0, 517, 505, 508, 528, 551, 529, 516, 556, 2, 0, 0, 0, 0, 0, 0, 0, 543, 501, 507, 495, 514, 479, 486, 506, 17, 7, 2, 0, 0, 0, 0, 0, 512, 534, 490, 517, 536, 520, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 509, 511, 452, 504, 530, 502, 518, 552, 0, 0, 0, 0, 0, 0, 0, 0, 520, 501, 494, 512, 497, 530, 506, 521, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4014, 4046, 4212, 4057, 4125, 4078, 4109] avg=4096.0000 max=4212 max_rank=3 min=4014 min_rank=1 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[529, 477, 482, 519, 522, 503, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 518, 526, 493, 508, 511, 528, 508, 502, 0, 0, 0, 0, 0, 0, 0, 0, 543, 506, 523, 489, 516, 501, 515, 513, 7, 2, 0, 0, 0, 0, 0, 0, 503, 521, 471, 509, 491, 504, 507, 526, 2, 0, 0, 0, 0, 0, 0, 0, 526, 505, 474, 543, 495, 501, 500, 569, 17, 8, 2, 0, 0, 0, 0, 0, 506, 541, 490, 543, 518, 481, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0, 491, 525, 508, 510, 497, 505, 542, 503, 0, 0, 0, 0, 0, 0, 0, 0, 533, 510, 520, 460, 514, 532, 505, 490, 28, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4094, 4115, 4034, 4140, 4128, 4081, 4092] avg=4096.0000 max=4140 max_rank=4 min=4034 min_rank=3 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[522, 532, 494, 507, 473, 555, 527, 497, 0, 0, 0, 0, 0, 0, 0, 0, 539, 497, 501, 514, 506, 553, 495, 537, 0, 0, 0, 0, 0, 0, 0, 0, 493, 539, 461, 505, 499, 480, 510, 518, 7, 1, 0, 0, 0, 0, 0, 0, 547, 462, 527, 532, 494, 492, 573, 524, 2, 0, 0, 0, 0, 0, 0, 0, 487, 489, 519, 497, 510, 538, 511, 509, 18, 7, 2, 0, 0, 0, 0, 0, 481, 474, 517, 457, 496, 522, 515, 514, 0, 0, 0, 0, 0, 0, 0, 0, 486, 519, 519, 522, 510, 539, 515, 534, 0, 0, 0, 0, 0, 0, 0, 0, 532, 492, 509, 529, 533, 527, 502, 495, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4142, 4013, 4153, 4087, 3976, 4144, 4146] avg=4096.0000 max=4153 max_rank=3 min=3976 min_rank=5 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 535, 487, 497, 515, 476, 519, 543, 0, 0, 0, 0, 0, 0, 0, 0, 509, 498, 544, 490, 505, 502, 493, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 528, 493, 494, 509, 542, 516, 502, 6, 1, 0, 0, 0, 0, 0, 0, 498, 502, 503, 533, 494, 519, 489, 537, 2, 0, 0, 0, 0, 0, 0, 0, 521, 499, 522, 541, 518, 516, 470, 491, 18, 7, 2, 0, 0, 0, 0, 0, 546, 531, 529, 551, 488, 515, 486, 514, 0, 0, 0, 0, 0, 0, 0, 0, 513, 488, 545, 520, 487, 545, 493, 518, 0, 0, 0, 0, 0, 0, 0, 0, 499, 499, 511, 451, 524, 528, 517, 508, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4042, 4127, 4077, 4105, 4160, 4109, 4064] avg=4096.0000 max=4160 max_rank=5 min=4042 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[519, 526, 528, 521, 508, 501, 506, 501, 0, 0, 0, 0, 0, 0, 0, 0, 502, 500, 531, 494, 496, 500, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 519, 509, 474, 508, 514, 535, 532, 6, 2, 0, 0, 0, 0, 0, 0, 542, 541, 544, 522, 476, 506, 532, 529, 2, 0, 0, 0, 0, 0, 0, 0, 476, 522, 515, 506, 493, 526, 492, 516, 17, 8, 2, 0, 0, 0, 0, 0, 521, 498, 503, 502, 525, 571, 516, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 480, 508, 464, 503, 480, 527, 0, 0, 0, 0, 0, 0, 0, 0, 513, 531, 485, 498, 538, 525, 508, 500, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4068, 4098, 4194, 4073, 4126, 3974, 4125] avg=4096.0000 max=4194 max_rank=3 min=3974 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[476, 552, 552, 539, 504, 521, 524, 485, 0, 0, 0, 0, 0, 0, 0, 0, 502, 509, 502, 504, 510, 522, 526, 529, 0, 0, 0, 0, 0, 0, 0, 0, 503, 559, 486, 513, 542, 516, 491, 512, 7, 2, 0, 0, 0, 0, 0, 0, 495, 515, 480, 546, 506, 520, 520, 515, 2, 0, 0, 0, 0, 0, 0, 0, 504, 507, 500, 527, 493, 515, 484, 474, 16, 8, 2, 0, 0, 0, 0, 0, 539, 550, 474, 547, 522, 469, 521, 499, 0, 0, 0, 0, 0, 0, 0, 0, 492, 516, 505, 530, 484, 505, 535, 526, 0, 0, 0, 0, 0, 0, 0, 0, 499, 479, 499, 533, 483, 489, 499, 529, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4104, 4131, 4099, 4030, 4121, 4093, 4037] avg=4096.0000 max=4153 max_rank=0 min=4030 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[521, 516, 503, 503, 513, 487, 487, 510, 0, 0, 0, 0, 0, 0, 0, 0, 555, 488, 540, 521, 543, 508, 504, 529, 0, 0, 0, 0, 0, 0, 0, 0, 552, 534, 523, 556, 489, 472, 529, 502, 10, 5, 0, 0, 0, 0, 0, 0, 510, 497, 507, 531, 512, 494, 484, 502, 7, 0, 0, 0, 0, 0, 0, 0, 482, 543, 525, 481, 542, 542, 497, 483, 20, 11, 3, 0, 0, 0, 0, 0, 480, 510, 515, 514, 495, 512, 549, 497, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 529, 511, 529, 495, 538, 471, 0, 0, 0, 0, 0, 0, 0, 0, 502, 514, 477, 518, 536, 494, 489, 484, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4188, 4172, 4044, 4129, 4072, 4079, 4044] avg=4096.0000 max=4188 max_rank=1 min=4040 min_rank=0 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[517, 556, 526, 475, 517, 511, 514, 478, 500, 512, 506, 498, 528, 528, 488, 524, 503, 474, 514, 511, 541, 496, 489, 553, 506, 536, 473, 565, 581, 518, 514, 543, 491, 487, 518, 532, 501, 517, 496, 526, 491, 539, 500, 487, 470, 556, 556, 481, 503, 516, 493, 501, 531, 531, 519, 497, 496, 524, 509, 494, 496, 522, 488, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[522, 503, 549, 546, 501, 468, 515, 509, 542, 523, 535, 473, 508, 512, 508, 542, 520, 525, 498, 522, 506, 504, 532, 502, 491, 530, 534, 490, 463, 521, 484, 516, 487, 505, 547, 528, 495, 522, 507, 539, 506, 500, 519, 537, 519, 480, 496, 484, 555, 514, 508, 467, 505, 484, 535, 513, 522, 484, 494, 475, 557, 521, 521, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[503, 530, 505, 536, 498, 486, 526, 508, 513, 515, 507, 493, 524, 483, 506, 555, 501, 509, 527, 549, 495, 532, 553, 504, 495, 543, 452, 512, 539, 510, 512, 494, 520, 523, 536, 533, 533, 510, 568, 499, 484, 517, 477, 508, 514, 481, 505, 508, 506, 547, 519, 520, 501, 477, 503, 495, 528, 499, 497, 506, 507, 520, 514, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[514, 488, 529, 518, 521, 513, 489, 523, 517, 495, 524, 472, 507, 489, 534, 502, 504, 502, 547, 529, 480, 517, 524, 520, 583, 514, 526, 512, 503, 498, 532, 548, 515, 491, 550, 523, 520, 489, 512, 494, 508, 478, 517, 507, 525, 532, 522, 498, 471, 507, 481, 517, 516, 488, 511, 516, 507, 519, 511, 518, 512, 512, 484, 543] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[543, 491, 548, 559, 509, 525, 558, 521, 530, 493, 487, 487, 531, 491, 501, 495, 468, 518, 501, 489, 545, 519, 497, 507, 493, 513, 538, 496, 519, 490, 518, 514, 524, 539, 496, 509, 516, 505, 518, 473, 485, 539, 558, 494, 507, 546, 521, 494, 523, 505, 517, 532, 502, 531, 515, 509, 500, 494, 507, 503, 490, 511, 508, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[537, 501, 500, 520, 556, 481, 496, 511, 529, 506, 483, 558, 520, 517, 502, 527, 499, 562, 524, 507, 520, 515, 473, 515, 524, 513, 492, 509, 558, 510, 532, 514, 501, 517, 502, 479, 513, 503, 479, 518, 486, 544, 501, 544, 477, 500, 501, 497, 499, 509, 543, 529, 512, 513, 513, 505, 489, 518, 531, 517, 467, 488, 525, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[503, 462, 539, 515, 515, 496, 534, 499, 495, 480, 554, 526, 511, 518, 510, 517, 522, 527, 504, 489, 533, 514, 512, 482, 506, 525, 491, 543, 542, 502, 518, 514, 491, 525, 503, 488, 502, 503, 492, 547, 504, 510, 534, 508, 510, 493, 511, 513, 518, 510, 506, 497, 537, 551, 493, 513, 528, 529, 535, 488, 522, 507, 482, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[521, 482, 539, 498, 526, 493, 483, 535, 515, 507, 529, 491, 495, 492, 506, 497, 527, 510, 552, 512, 528, 509, 569, 497, 505, 509, 526, 516, 530, 495, 512, 497, 500, 510, 482, 499, 485, 527, 515, 479, 514, 512, 533, 504, 486, 522, 544, 547, 530, 501, 544, 493, 509, 487, 478, 544, 525, 523, 517, 521, 530, 513, 498, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 556, 512, 475, 517, 511, 514, 478, 0, 0, 0, 0, 0, 0, 0, 0, 500, 512, 506, 498, 528, 528, 488, 524, 8, 4, 0, 0, 0, 0, 0, 0, 503, 474, 495, 511, 541, 496, 489, 553, 0, 0, 0, 0, 0, 0, 0, 0, 506, 536, 473, 565, 550, 518, 514, 543, 0, 0, 0, 0, 0, 0, 0, 0, 491, 487, 518, 532, 501, 517, 496, 526, 14, 0, 0, 0, 0, 0, 0, 0, 491, 539, 500, 487, 470, 556, 556, 481, 17, 0, 0, 0, 0, 0, 0, 0, 503, 516, 493, 501, 531, 531, 519, 497, 6, 0, 0, 0, 0, 0, 0, 0, 496, 524, 509, 494, 496, 522, 488, 505, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4096, 4062, 4205, 4082, 4097, 4097, 4049] avg=4096.0000 max=4205 max_rank=3 min=4049 min_rank=7 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[522, 503, 535, 546, 501, 468, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 542, 523, 535, 473, 508, 512, 508, 542, 8, 4, 0, 0, 0, 0, 0, 0, 520, 525, 480, 522, 506, 504, 532, 502, 0, 0, 0, 0, 0, 0, 0, 0, 491, 530, 534, 490, 439, 521, 484, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 547, 528, 495, 522, 507, 539, 13, 0, 0, 0, 0, 0, 0, 0, 506, 500, 519, 537, 519, 480, 496, 484, 13, 0, 0, 0, 0, 0, 0, 0, 555, 514, 508, 467, 505, 484, 535, 513, 6, 0, 0, 0, 0, 0, 0, 0, 522, 484, 494, 475, 557, 521, 521, 548, 11, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4155, 4091, 4005, 4143, 4054, 4087, 4134] avg=4096.0000 max=4155 max_rank=1 min=4005 min_rank=3 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 530, 491, 536, 498, 486, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 507, 493, 524, 483, 506, 555, 8, 4, 0, 0, 0, 0, 0, 0, 501, 509, 508, 549, 495, 532, 553, 504, 0, 0, 0, 0, 0, 0, 0, 0, 495, 543, 452, 512, 510, 510, 512, 494, 0, 0, 0, 0, 0, 0, 0, 0, 520, 523, 536, 533, 533, 510, 568, 499, 14, 0, 0, 0, 0, 0, 0, 0, 484, 517, 477, 508, 514, 481, 505, 508, 16, 0, 0, 0, 0, 0, 0, 0, 506, 547, 519, 520, 501, 477, 503, 495, 6, 0, 0, 0, 0, 0, 0, 0, 528, 499, 497, 506, 507, 520, 514, 498, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4108, 4151, 4028, 4236, 4010, 4074, 4083] avg=4096.0000 max=4236 max_rank=4 min=4010 min_rank=5 imbalance_ratio=1.034180 max_load_reduction=28532 max_load_reduction_pct=0.870728 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[543, 491, 534, 559, 509, 525, 558, 521, 0, 0, 0, 0, 0, 0, 0, 0, 530, 493, 487, 487, 531, 491, 501, 495, 8, 4, 0, 0, 0, 0, 0, 0, 468, 518, 483, 489, 545, 519, 497, 507, 0, 0, 0, 0, 0, 0, 0, 0, 493, 513, 538, 496, 491, 490, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 524, 539, 496, 509, 516, 505, 518, 473, 13, 0, 0, 0, 0, 0, 0, 0, 485, 539, 558, 494, 507, 546, 521, 494, 15, 0, 0, 0, 0, 0, 0, 0, 523, 505, 517, 532, 502, 531, 515, 509, 6, 0, 0, 0, 0, 0, 0, 0, 500, 494, 507, 503, 490, 511, 508, 503, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4240, 4027, 4026, 4053, 4093, 4159, 4140, 4030] avg=4096.0000 max=4240 max_rank=0 min=4026 min_rank=2 imbalance_ratio=1.035156 max_load_reduction=28528 max_load_reduction_pct=0.870605 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 482, 525, 498, 526, 493, 483, 535, 0, 0, 0, 0, 0, 0, 0, 0, 515, 507, 529, 491, 495, 492, 506, 497, 8, 4, 0, 0, 0, 0, 0, 0, 527, 510, 532, 512, 528, 509, 569, 497, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 526, 516, 502, 495, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 500, 510, 482, 499, 485, 527, 515, 479, 15, 0, 0, 0, 0, 0, 0, 0, 514, 512, 533, 504, 486, 522, 544, 547, 15, 0, 0, 0, 0, 0, 0, 0, 530, 501, 544, 493, 509, 487, 478, 544, 6, 0, 0, 0, 0, 0, 0, 0, 525, 523, 517, 521, 530, 513, 498, 493, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4063, 4044, 4184, 4062, 4012, 4177, 4092, 4134] avg=4096.0000 max=4184 max_rank=2 min=4012 min_rank=4 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[537, 501, 486, 520, 556, 481, 496, 511, 0, 0, 0, 0, 0, 0, 0, 0, 529, 506, 483, 558, 520, 517, 502, 527, 8, 4, 0, 0, 0, 0, 0, 0, 499, 562, 505, 507, 520, 515, 473, 515, 0, 0, 0, 0, 0, 0, 0, 0, 524, 513, 492, 509, 528, 510, 532, 514, 0, 0, 0, 0, 0, 0, 0, 0, 501, 517, 502, 479, 513, 503, 479, 518, 14, 0, 0, 0, 0, 0, 0, 0, 486, 544, 501, 544, 477, 500, 501, 497, 16, 0, 0, 0, 0, 0, 0, 0, 499, 509, 543, 529, 512, 513, 513, 505, 6, 0, 0, 0, 0, 0, 0, 0, 489, 518, 531, 517, 467, 488, 525, 537, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4154, 4096, 4122, 4026, 4066, 4129, 4087] avg=4096.0000 max=4154 max_rank=1 min=4026 min_rank=4 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[514, 488, 515, 518, 521, 513, 489, 523, 0, 0, 0, 0, 0, 0, 0, 0, 517, 495, 524, 472, 507, 489, 534, 502, 8, 4, 0, 0, 0, 0, 0, 0, 504, 502, 528, 529, 480, 517, 524, 520, 0, 0, 0, 0, 0, 0, 0, 0, 583, 514, 526, 512, 476, 498, 532, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 491, 550, 523, 520, 489, 512, 494, 14, 0, 0, 0, 0, 0, 0, 0, 508, 478, 517, 507, 525, 532, 522, 498, 15, 0, 0, 0, 0, 0, 0, 0, 471, 507, 481, 517, 516, 488, 511, 516, 6, 0, 0, 0, 0, 0, 0, 0, 507, 519, 511, 518, 512, 512, 484, 543, 12, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4052, 4104, 4189, 4108, 4102, 4013, 4119] avg=4096.0000 max=4189 max_rank=3 min=4013 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16342 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[503, 462, 515, 515, 515, 496, 534, 499, 0, 0, 0, 0, 0, 0, 0, 0, 495, 480, 554, 526, 511, 518, 510, 517, 13, 8, 0, 0, 0, 0, 0, 0, 522, 527, 475, 489, 533, 514, 512, 482, 0, 0, 0, 0, 0, 0, 0, 0, 506, 525, 491, 543, 505, 502, 518, 514, 0, 0, 0, 0, 0, 0, 0, 0, 491, 525, 503, 488, 502, 503, 492, 547, 17, 0, 0, 0, 0, 0, 0, 0, 504, 510, 534, 508, 510, 493, 511, 513, 20, 0, 0, 0, 0, 0, 0, 0, 518, 510, 506, 497, 537, 551, 493, 513, 11, 0, 0, 0, 0, 0, 0, 0, 528, 529, 535, 488, 522, 507, 482, 520, 17, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4132, 4054, 4104, 4068, 4103, 4136, 4132] avg=4096.0000 max=4136 max_rank=6 min=4039 min_rank=0 imbalance_ratio=1.009766 max_load_reduction=28632 max_load_reduction_pct=0.873779 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[499, 529, 519, 464, 520, 505, 516, 509, 512, 497, 554, 511, 525, 516, 499, 518, 469, 518, 538, 495, 534, 491, 520, 504, 484, 491, 546, 532, 530, 498, 553, 511, 519, 505, 526, 512, 502, 499, 503, 547, 498, 538, 535, 557, 538, 469, 489, 530, 479, 538, 525, 494, 540, 498, 488, 509, 524, 513, 488, 515, 491, 488, 510, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[510, 486, 526, 552, 499, 505, 517, 499, 499, 491, 482, 492, 510, 514, 501, 507, 524, 510, 534, 505, 537, 493, 528, 498, 554, 518, 495, 554, 499, 504, 548, 518, 516, 491, 543, 516, 493, 513, 494, 523, 474, 517, 493, 512, 512, 490, 514, 508, 517, 494, 520, 531, 516, 484, 501, 515, 546, 487, 534, 519, 519, 553, 458, 556] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[496, 506, 503, 533, 514, 527, 529, 506, 521, 486, 482, 494, 527, 539, 477, 523, 499, 507, 503, 535, 544, 485, 512, 514, 499, 547, 523, 506, 474, 488, 502, 532, 498, 540, 515, 505, 513, 550, 537, 501, 483, 479, 505, 491, 543, 535, 511, 520, 535, 551, 487, 539, 465, 488, 479, 496, 551, 510, 503, 519, 548, 500, 537, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[505, 532, 495, 505, 552, 514, 491, 542, 492, 479, 508, 519, 533, 527, 533, 496, 519, 505, 535, 519, 488, 545, 499, 526, 527, 546, 490, 507, 533, 545, 448, 505, 511, 490, 504, 503, 501, 507, 511, 532, 511, 510, 548, 529, 500, 518, 477, 523, 516, 451, 491, 507, 494, 516, 535, 503, 485, 521, 524, 503, 542, 520, 557, 468] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[498, 507, 520, 504, 493, 513, 529, 512, 552, 488, 519, 529, 512, 536, 514, 500, 532, 538, 456, 516, 511, 538, 516, 533, 488, 500, 556, 499, 494, 509, 526, 509, 509, 487, 492, 487, 502, 521, 480, 521, 506, 562, 526, 508, 498, 550, 513, 529, 510, 524, 490, 531, 512, 518, 485, 481, 485, 512, 541, 480, 517, 543, 508, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[511, 523, 493, 523, 513, 470, 497, 536, 525, 512, 514, 518, 483, 548, 529, 540, 483, 535, 519, 521, 487, 481, 554, 512, 534, 482, 487, 535, 499, 514, 520, 512, 497, 493, 538, 514, 500, 541, 527, 496, 536, 543, 516, 543, 533, 514, 508, 481, 512, 550, 507, 501, 502, 498, 520, 509, 470, 529, 511, 519, 497, 485, 484, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[510, 492, 555, 526, 529, 472, 508, 513, 505, 524, 493, 534, 527, 474, 473, 516, 489, 511, 530, 513, 565, 513, 535, 487, 550, 498, 514, 534, 494, 504, 476, 517, 490, 487, 562, 511, 526, 512, 534, 499, 570, 533, 503, 500, 531, 472, 530, 482, 547, 484, 530, 488, 487, 492, 533, 506, 483, 534, 534, 536, 491, 496, 496, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[503, 512, 515, 490, 512, 502, 518, 510, 499, 562, 537, 465, 510, 518, 536, 506, 530, 503, 485, 495, 547, 499, 515, 560, 475, 500, 542, 482, 536, 537, 500, 531, 527, 476, 487, 537, 506, 534, 492, 528, 486, 515, 496, 505, 496, 543, 533, 487, 517, 506, 527, 546, 535, 489, 528, 530, 500, 516, 537, 502, 460, 509, 495, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 529, 519, 464, 520, 505, 516, 509, 6, 0, 0, 0, 0, 0, 0, 0, 512, 497, 554, 511, 525, 516, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 469, 518, 538, 495, 513, 491, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 484, 491, 536, 532, 530, 498, 553, 511, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 526, 512, 502, 499, 503, 547, 4, 0, 0, 0, 0, 0, 0, 0, 498, 521, 535, 557, 538, 469, 489, 530, 0, 0, 0, 0, 0, 0, 0, 0, 479, 538, 525, 494, 540, 498, 488, 509, 21, 0, 0, 0, 0, 0, 0, 0, 524, 513, 488, 515, 491, 488, 510, 494, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4132, 4048, 4135, 4117, 4137, 4092, 4040] avg=4096.0000 max=4137 max_rank=5 min=4040 min_rank=7 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[505, 532, 495, 505, 552, 514, 491, 542, 6, 0, 0, 0, 0, 0, 0, 0, 492, 479, 508, 519, 533, 527, 533, 496, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 535, 519, 469, 545, 499, 526, 0, 0, 0, 0, 0, 0, 0, 0, 527, 546, 480, 507, 533, 545, 448, 505, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 504, 503, 501, 507, 511, 532, 4, 0, 0, 0, 0, 0, 0, 0, 511, 494, 548, 529, 500, 518, 477, 523, 0, 0, 0, 0, 0, 0, 0, 0, 516, 451, 491, 507, 494, 516, 535, 503, 19, 0, 0, 0, 0, 0, 0, 0, 485, 521, 524, 503, 542, 520, 557, 468, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4087, 4117, 4091, 4063, 4100, 4032, 4136] avg=4096.0000 max=4142 max_rank=0 min=4032 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16331 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[496, 506, 503, 533, 514, 527, 529, 506, 6, 0, 0, 0, 0, 0, 0, 0, 521, 486, 482, 494, 527, 539, 477, 523, 0, 0, 0, 0, 0, 0, 0, 0, 499, 507, 503, 535, 522, 485, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 499, 547, 513, 506, 474, 488, 502, 532, 0, 0, 0, 0, 0, 0, 0, 0, 498, 540, 515, 505, 513, 550, 537, 501, 4, 0, 0, 0, 0, 0, 0, 0, 483, 464, 505, 491, 543, 535, 511, 520, 0, 0, 0, 0, 0, 0, 0, 0, 535, 551, 487, 539, 465, 488, 479, 496, 22, 0, 0, 0, 0, 0, 0, 0, 551, 510, 503, 519, 548, 500, 537, 501, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4049, 4077, 4061, 4163, 4052, 4062, 4184] avg=4096.0000 max=4184 max_rank=7 min=4049 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16461 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 486, 526, 552, 499, 505, 517, 499, 6, 0, 0, 0, 0, 0, 0, 0, 499, 491, 482, 492, 510, 514, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 524, 510, 534, 505, 516, 493, 528, 498, 0, 0, 0, 0, 0, 0, 0, 0, 554, 518, 485, 554, 499, 504, 548, 518, 0, 0, 0, 0, 0, 0, 0, 0, 516, 491, 543, 516, 493, 513, 494, 523, 4, 0, 0, 0, 0, 0, 0, 0, 474, 501, 493, 512, 512, 490, 514, 508, 0, 0, 0, 0, 0, 0, 0, 0, 517, 494, 520, 531, 516, 484, 501, 515, 21, 0, 0, 0, 0, 0, 0, 0, 546, 487, 534, 519, 519, 553, 458, 556, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 3996, 4108, 4180, 4093, 4004, 4099, 4188] avg=4096.0000 max=4188 max_rank=7 min=3996 min_rank=1 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[511, 523, 493, 523, 513, 470, 497, 536, 6, 0, 0, 0, 0, 0, 0, 0, 525, 512, 514, 518, 483, 548, 529, 540, 0, 0, 0, 0, 0, 0, 0, 0, 483, 535, 519, 521, 468, 481, 554, 512, 0, 0, 0, 0, 0, 0, 0, 0, 534, 482, 477, 535, 499, 514, 520, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 538, 514, 500, 541, 527, 496, 4, 0, 0, 0, 0, 0, 0, 0, 536, 526, 516, 543, 533, 514, 508, 481, 0, 0, 0, 0, 0, 0, 0, 0, 512, 550, 507, 501, 502, 498, 520, 509, 19, 0, 0, 0, 0, 0, 0, 0, 470, 529, 511, 519, 497, 485, 484, 484, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4072, 4169, 4073, 4073, 4110, 4157, 4118, 3996] avg=4096.0000 max=4169 max_rank=1 min=3996 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[510, 492, 555, 526, 529, 472, 508, 513, 6, 0, 0, 0, 0, 0, 0, 0, 505, 524, 493, 534, 527, 474, 473, 516, 0, 0, 0, 0, 0, 0, 0, 0, 489, 511, 530, 513, 543, 513, 535, 487, 0, 0, 0, 0, 0, 0, 0, 0, 550, 498, 504, 534, 494, 504, 476, 517, 0, 0, 0, 0, 0, 0, 0, 0, 490, 487, 562, 511, 526, 512, 534, 499, 4, 0, 0, 0, 0, 0, 0, 0, 570, 516, 503, 500, 531, 472, 530, 482, 0, 0, 0, 0, 0, 0, 0, 0, 547, 484, 530, 488, 487, 492, 533, 506, 22, 0, 0, 0, 0, 0, 0, 0, 483, 534, 534, 536, 491, 496, 496, 508, 17, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4046, 4121, 4077, 4125, 4104, 4089, 4095] avg=4096.0000 max=4125 max_rank=4 min=4046 min_rank=1 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[498, 507, 520, 504, 493, 513, 529, 512, 10, 2, 0, 0, 0, 0, 0, 0, 552, 488, 519, 529, 512, 536, 514, 500, 6, 0, 0, 0, 0, 0, 0, 0, 532, 538, 456, 516, 483, 538, 516, 533, 0, 0, 0, 0, 0, 0, 0, 0, 488, 500, 533, 499, 494, 509, 526, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 487, 492, 487, 502, 521, 480, 521, 7, 0, 0, 0, 0, 0, 0, 0, 506, 539, 526, 508, 498, 550, 513, 529, 0, 0, 0, 0, 0, 0, 0, 0, 510, 524, 490, 531, 512, 518, 485, 481, 25, 0, 0, 0, 0, 0, 0, 0, 485, 512, 541, 480, 517, 543, 508, 493, 21, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4156, 4112, 4058, 4006, 4169, 4076, 4103] avg=4096.0000 max=4169 max_rank=5 min=4006 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[503, 512, 515, 490, 512, 502, 518, 510, 6, 0, 0, 0, 0, 0, 0, 0, 499, 562, 537, 465, 510, 518, 536, 506, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 485, 495, 525, 499, 515, 560, 0, 0, 0, 0, 0, 0, 0, 0, 475, 500, 532, 482, 536, 537, 500, 531, 0, 0, 0, 0, 0, 0, 0, 0, 527, 476, 487, 537, 506, 534, 492, 528, 4, 0, 0, 0, 0, 0, 0, 0, 486, 499, 496, 505, 496, 543, 533, 487, 0, 0, 0, 0, 0, 0, 0, 0, 517, 506, 527, 546, 535, 489, 528, 530, 22, 0, 0, 0, 0, 0, 0, 0, 500, 516, 537, 502, 460, 509, 495, 491, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4133, 4112, 4093, 4091, 4045, 4200, 4026] avg=4096.0000 max=4200 max_rank=6 min=4026 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[513, 511, 535, 519, 536, 549, 533, 524, 504, 489, 516, 488, 516, 501, 504, 504, 537, 535, 516, 508, 528, 510, 511, 517, 496, 488, 481, 503, 522, 531, 534, 499, 505, 511, 504, 511, 464, 549, 489, 523, 526, 535, 512, 520, 483, 520, 541, 523, 518, 493, 488, 510, 518, 575, 541, 503, 511, 469, 497, 486, 466, 515, 471, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[500, 514, 542, 486, 545, 509, 551, 521, 502, 501, 498, 491, 474, 512, 545, 507, 511, 497, 513, 502, 531, 501, 484, 515, 546, 474, 498, 518, 466, 525, 459, 542, 513, 498, 505, 540, 536, 529, 518, 499, 516, 520, 523, 503, 517, 543, 551, 463, 516, 509, 521, 515, 521, 527, 509, 512, 499, 548, 526, 512, 498, 504, 491, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[499, 501, 509, 509, 495, 507, 523, 498, 483, 467, 501, 513, 508, 522, 503, 573, 566, 520, 520, 518, 531, 501, 497, 535, 525, 515, 448, 571, 525, 488, 495, 491, 533, 500, 523, 490, 506, 511, 524, 503, 492, 541, 519, 500, 505, 516, 467, 510, 508, 497, 514, 527, 532, 525, 474, 475, 539, 520, 541, 518, 563, 496, 517, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[514, 515, 508, 497, 516, 535, 499, 520, 491, 495, 562, 537, 504, 529, 529, 491, 511, 513, 510, 474, 574, 520, 481, 526, 517, 509, 486, 486, 518, 482, 507, 548, 527, 485, 521, 521, 529, 510, 516, 504, 536, 501, 498, 484, 534, 536, 513, 476, 542, 505, 528, 490, 508, 493, 483, 513, 528, 516, 499, 537, 515, 538, 503, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[553, 525, 520, 546, 498, 532, 500, 489, 517, 492, 515, 481, 489, 516, 540, 482, 540, 526, 552, 505, 538, 543, 504, 517, 537, 537, 517, 516, 529, 527, 507, 538, 481, 461, 485, 481, 503, 479, 517, 531, 502, 505, 547, 501, 524, 507, 496, 491, 462, 471, 489, 510, 545, 502, 524, 504, 521, 473, 476, 514, 538, 539, 539, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[511, 538, 485, 545, 540, 511, 503, 528, 485, 539, 542, 546, 522, 533, 519, 522, 494, 529, 480, 576, 490, 545, 520, 504, 499, 509, 493, 487, 519, 552, 506, 514, 526, 533, 481, 489, 517, 497, 493, 539, 519, 490, 526, 467, 495, 488, 488, 521, 525, 501, 521, 501, 512, 499, 524, 508, 496, 470, 529, 536, 467, 508, 491, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[506, 495, 514, 571, 537, 516, 519, 542, 511, 498, 545, 520, 522, 506, 493, 519, 492, 507, 490, 498, 541, 562, 523, 519, 523, 524, 533, 541, 548, 493, 465, 489, 481, 499, 537, 486, 506, 498, 533, 487, 496, 531, 511, 517, 456, 487, 504, 527, 490, 524, 542, 528, 500, 521, 465, 527, 549, 496, 497, 481, 465, 545, 551, 469] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[510, 490, 501, 526, 546, 524, 524, 538, 525, 520, 539, 495, 559, 533, 506, 484, 535, 526, 510, 500, 520, 513, 510, 492, 509, 491, 513, 516, 486, 498, 543, 495, 492, 507, 515, 534, 521, 472, 544, 510, 486, 496, 528, 489, 503, 515, 516, 515, 526, 512, 481, 531, 530, 490, 506, 515, 514, 515, 497, 494, 506, 524, 534, 473] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 511, 535, 519, 480, 549, 533, 524, 0, 0, 0, 0, 0, 0, 0, 0, 504, 489, 516, 488, 516, 501, 504, 504, 0, 0, 0, 0, 0, 0, 0, 0, 537, 535, 516, 508, 486, 510, 511, 517, 0, 0, 0, 0, 0, 0, 0, 0, 496, 488, 481, 503, 522, 531, 534, 499, 15, 0, 0, 0, 0, 0, 0, 0, 505, 511, 504, 511, 464, 549, 489, 523, 30, 0, 0, 0, 0, 0, 0, 0, 526, 535, 512, 520, 483, 520, 541, 523, 25, 0, 0, 0, 0, 0, 0, 0, 518, 493, 488, 510, 518, 575, 541, 503, 10, 0, 0, 0, 0, 0, 0, 0, 511, 469, 497, 486, 466, 515, 471, 533, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4022, 4120, 4069, 4086, 4185, 4156, 3966] avg=4096.0000 max=4185 max_rank=5 min=3966 min_rank=7 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[553, 525, 520, 546, 447, 532, 500, 489, 0, 0, 0, 0, 0, 0, 0, 0, 517, 492, 515, 481, 489, 516, 540, 482, 0, 0, 0, 0, 0, 0, 0, 0, 540, 526, 552, 505, 496, 543, 504, 517, 0, 0, 0, 0, 0, 0, 0, 0, 537, 537, 517, 516, 529, 527, 507, 538, 15, 0, 0, 0, 0, 0, 0, 0, 481, 461, 485, 481, 503, 479, 517, 531, 27, 0, 0, 0, 0, 0, 0, 0, 502, 505, 547, 501, 524, 507, 496, 491, 23, 0, 0, 0, 0, 0, 0, 0, 462, 471, 489, 510, 545, 502, 524, 504, 10, 0, 0, 0, 0, 0, 0, 0, 521, 473, 476, 514, 538, 539, 539, 522, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4032, 4183, 4223, 3965, 4096, 4017, 4140] avg=4096.0000 max=4223 max_rank=3 min=3965 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16218 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 514, 542, 486, 489, 509, 551, 521, 0, 0, 0, 0, 0, 0, 0, 0, 502, 501, 498, 491, 474, 512, 545, 507, 0, 0, 0, 0, 0, 0, 0, 0, 511, 497, 513, 502, 489, 501, 484, 515, 0, 0, 0, 0, 0, 0, 0, 0, 546, 474, 498, 518, 466, 525, 459, 542, 15, 0, 0, 0, 0, 0, 0, 0, 513, 498, 505, 540, 536, 529, 518, 499, 30, 0, 0, 0, 0, 0, 0, 0, 516, 520, 523, 503, 517, 543, 551, 463, 25, 0, 0, 0, 0, 0, 0, 0, 516, 509, 521, 515, 521, 527, 509, 512, 10, 0, 0, 0, 0, 0, 0, 0, 499, 548, 526, 512, 498, 504, 491, 506, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4112, 4030, 4012, 4043, 4168, 4161, 4140, 4102] avg=4096.0000 max=4168 max_rank=4 min=4012 min_rank=2 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16571 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[514, 515, 508, 497, 463, 535, 499, 520, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 562, 537, 504, 529, 529, 491, 0, 0, 0, 0, 0, 0, 0, 0, 511, 513, 510, 474, 529, 520, 481, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 486, 486, 518, 482, 507, 548, 16, 0, 0, 0, 0, 0, 0, 0, 527, 485, 521, 521, 529, 510, 516, 504, 28, 0, 0, 0, 0, 0, 0, 0, 536, 501, 498, 484, 534, 536, 513, 476, 24, 0, 0, 0, 0, 0, 0, 0, 542, 505, 528, 490, 508, 493, 483, 513, 11, 0, 0, 0, 0, 0, 0, 0, 528, 516, 499, 537, 515, 538, 503, 475, 18, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4138, 4064, 4069, 4141, 4102, 4073, 4130] avg=4096.0000 max=4141 max_rank=4 min=4051 min_rank=0 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[499, 501, 509, 509, 444, 507, 523, 498, 0, 0, 0, 0, 0, 0, 0, 0, 483, 467, 501, 513, 508, 522, 503, 573, 0, 0, 0, 0, 0, 0, 0, 0, 566, 520, 520, 518, 489, 501, 497, 535, 0, 0, 0, 0, 0, 0, 0, 0, 525, 515, 448, 571, 525, 488, 495, 491, 15, 0, 0, 0, 0, 0, 0, 0, 533, 500, 523, 490, 506, 511, 524, 503, 27, 0, 0, 0, 0, 0, 0, 0, 492, 541, 519, 500, 505, 516, 467, 510, 23, 0, 0, 0, 0, 0, 0, 0, 508, 497, 514, 527, 532, 525, 474, 475, 10, 0, 0, 0, 0, 0, 0, 0, 539, 520, 541, 518, 563, 496, 517, 525, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3990, 4070, 4146, 4073, 4117, 4073, 4062, 4237] avg=4096.0000 max=4237 max_rank=7 min=3990 min_rank=0 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16489 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[506, 495, 514, 571, 481, 516, 519, 542, 0, 0, 0, 0, 0, 0, 0, 0, 511, 498, 545, 520, 522, 506, 493, 519, 0, 0, 0, 0, 0, 0, 0, 0, 492, 507, 490, 498, 499, 562, 523, 519, 0, 0, 0, 0, 0, 0, 0, 0, 523, 524, 533, 541, 548, 493, 465, 489, 15, 0, 0, 0, 0, 0, 0, 0, 481, 499, 537, 486, 506, 498, 533, 487, 30, 0, 0, 0, 0, 0, 0, 0, 496, 531, 511, 517, 456, 487, 504, 527, 25, 0, 0, 0, 0, 0, 0, 0, 490, 524, 542, 528, 500, 521, 465, 527, 10, 0, 0, 0, 0, 0, 0, 0, 549, 496, 497, 481, 465, 545, 551, 469, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4144, 4114, 4090, 4131, 4057, 4054, 4107, 4071] avg=4096.0000 max=4144 max_rank=0 min=4054 min_rank=5 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16289 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 538, 485, 545, 471, 511, 503, 528, 0, 0, 0, 0, 0, 0, 0, 0, 485, 539, 535, 546, 522, 533, 519, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 529, 480, 576, 440, 545, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 499, 509, 493, 487, 519, 552, 506, 514, 16, 0, 0, 0, 0, 0, 0, 0, 526, 533, 481, 489, 517, 497, 493, 539, 34, 0, 0, 0, 0, 0, 0, 0, 519, 490, 526, 467, 495, 488, 488, 521, 30, 0, 0, 0, 0, 0, 0, 0, 525, 501, 521, 501, 512, 499, 524, 508, 14, 7, 0, 0, 0, 0, 0, 0, 496, 470, 529, 536, 467, 508, 491, 525, 20, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4201, 4088, 4095, 4109, 4024, 4112, 4047] avg=4096.0000 max=4201 max_rank=1 min=4024 min_rank=5 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[510, 490, 501, 526, 490, 524, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 525, 520, 539, 495, 559, 533, 506, 484, 0, 0, 0, 0, 0, 0, 0, 0, 535, 526, 510, 500, 479, 513, 510, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 513, 516, 486, 498, 543, 495, 14, 0, 0, 0, 0, 0, 0, 0, 492, 507, 515, 534, 521, 472, 544, 510, 30, 0, 0, 0, 0, 0, 0, 0, 486, 496, 528, 489, 503, 515, 516, 515, 25, 0, 0, 0, 0, 0, 0, 0, 526, 512, 481, 531, 530, 490, 506, 515, 10, 0, 0, 0, 0, 0, 0, 0, 514, 515, 497, 494, 506, 524, 534, 473, 17, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4161, 4065, 4065, 4125, 4073, 4101, 4075] avg=4096.0000 max=4161 max_rank=1 min=4065 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 3840, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[541, 498, 528, 547, 490, 504, 494, 512, 504, 492, 507, 515, 462, 485, 491, 543, 494, 501, 512, 515, 536, 524, 501, 540, 519, 543, 478, 523, 486, 503, 532, 500, 529, 502, 506, 498, 531, 509, 507, 524, 501, 518, 531, 544, 492, 549, 538, 511, 512, 517, 478, 497, 503, 487, 525, 505, 524, 526, 545, 491, 555, 484, 515, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[488, 515, 536, 504, 534, 533, 575, 528, 488, 546, 523, 538, 536, 538, 495, 532, 490, 550, 518, 537, 537, 540, 471, 495, 516, 507, 509, 570, 532, 510, 511, 559, 530, 507, 512, 488, 536, 520, 454, 488, 452, 485, 481, 490, 486, 464, 524, 507, 467, 506, 486, 518, 528, 523, 532, 532, 511, 518, 495, 494, 502, 506, 487, 478] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[493, 535, 516, 477, 492, 503, 505, 492, 520, 502, 534, 555, 520, 526, 525, 537, 508, 501, 459, 486, 545, 497, 503, 505, 529, 517, 522, 524, 508, 470, 524, 528, 525, 509, 519, 483, 497, 484, 523, 483, 538, 550, 525, 509, 500, 490, 576, 507, 545, 497, 527, 533, 525, 535, 498, 531, 510, 503, 470, 509, 506, 491, 491, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[542, 569, 511, 460, 486, 534, 489, 534, 545, 551, 538, 451, 512, 526, 510, 473, 522, 513, 524, 505, 549, 520, 532, 534, 498, 513, 541, 451, 527, 526, 532, 504, 512, 544, 500, 521, 515, 523, 504, 503, 492, 499, 547, 471, 504, 470, 499, 530, 468, 477, 509, 533, 513, 483, 530, 485, 506, 499, 516, 518, 528, 519, 516, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[482, 507, 509, 538, 488, 505, 505, 513, 557, 524, 506, 506, 466, 556, 545, 526, 492, 531, 508, 496, 542, 490, 513, 477, 493, 527, 524, 503, 517, 494, 535, 471, 522, 501, 499, 504, 510, 505, 506, 526, 497, 553, 531, 503, 517, 492, 553, 503, 482, 487, 516, 552, 494, 517, 516, 509, 529, 505, 540, 478, 531, 508, 538, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[516, 529, 566, 503, 518, 493, 532, 527, 484, 537, 501, 516, 478, 530, 529, 536, 469, 524, 496, 537, 536, 529, 509, 507, 515, 511, 500, 512, 552, 496, 495, 462, 522, 514, 505, 523, 520, 489, 490, 491, 502, 548, 543, 541, 519, 514, 526, 511, 508, 522, 514, 498, 527, 533, 491, 503, 517, 461, 473, 527, 483, 485, 519, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[506, 471, 522, 514, 519, 479, 564, 515, 525, 522, 504, 504, 494, 489, 535, 560, 474, 546, 486, 530, 478, 498, 528, 527, 489, 521, 522, 506, 509, 522, 502, 549, 517, 498, 514, 517, 500, 522, 498, 530, 543, 508, 511, 519, 521, 490, 460, 522, 456, 524, 507, 535, 526, 521, 479, 539, 545, 501, 498, 516, 541, 516, 494, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[522, 518, 517, 487, 478, 482, 522, 513, 527, 519, 511, 512, 537, 523, 506, 528, 548, 531, 528, 503, 519, 494, 518, 515, 518, 510, 506, 546, 529, 552, 498, 519, 479, 517, 519, 499, 538, 494, 504, 505, 505, 508, 458, 483, 493, 504, 505, 504, 500, 530, 531, 542, 510, 470, 477, 567, 480, 491, 522, 501, 554, 531, 506, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[541, 498, 521, 547, 490, 504, 494, 512, 0, 0, 0, 0, 0, 0, 0, 0, 504, 492, 507, 515, 462, 485, 491, 500, 0, 0, 0, 0, 0, 0, 0, 0, 494, 501, 512, 515, 528, 524, 501, 540, 0, 0, 0, 0, 0, 0, 0, 0, 519, 543, 478, 523, 470, 503, 532, 500, 0, 0, 0, 0, 0, 0, 0, 0, 529, 502, 506, 498, 531, 509, 507, 524, 13, 11, 0, 0, 0, 0, 0, 0, 501, 518, 531, 544, 492, 549, 538, 511, 6, 5, 0, 0, 0, 0, 0, 0, 512, 517, 478, 497, 503, 487, 525, 505, 7, 2, 0, 0, 0, 0, 0, 0, 524, 526, 545, 491, 555, 484, 515, 494, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 3956, 4115, 4068, 4130, 4195, 4033, 4164] avg=4096.0000 max=4195 max_rank=5 min=3956 min_rank=1 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16522 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 535, 510, 477, 492, 503, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 520, 502, 534, 555, 520, 526, 525, 494, 0, 0, 0, 0, 0, 0, 0, 0, 508, 501, 459, 486, 537, 497, 503, 505, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 522, 524, 491, 470, 524, 528, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 519, 483, 497, 484, 523, 483, 13, 12, 0, 0, 0, 0, 0, 0, 538, 550, 525, 509, 500, 490, 576, 507, 6, 5, 0, 0, 0, 0, 0, 0, 545, 497, 527, 533, 525, 535, 498, 531, 6, 2, 0, 0, 0, 0, 0, 0, 510, 503, 470, 509, 506, 491, 491, 521, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4176, 3996, 4105, 4048, 4206, 4199, 4031] avg=4096.0000 max=4206 max_rank=5 min=3996 min_rank=2 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[482, 507, 503, 538, 488, 505, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 557, 524, 506, 506, 466, 556, 545, 485, 0, 0, 0, 0, 0, 0, 0, 0, 492, 531, 508, 496, 534, 490, 513, 477, 0, 0, 0, 0, 0, 0, 0, 0, 493, 527, 524, 503, 499, 494, 535, 471, 0, 0, 0, 0, 0, 0, 0, 0, 522, 501, 499, 504, 510, 505, 506, 526, 12, 12, 0, 0, 0, 0, 0, 0, 497, 553, 531, 503, 517, 492, 553, 503, 6, 6, 0, 0, 0, 0, 0, 0, 482, 487, 516, 552, 494, 517, 516, 509, 6, 2, 0, 0, 0, 0, 0, 0, 529, 505, 540, 478, 531, 508, 538, 498, 29, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4041, 4145, 4041, 4046, 4097, 4161, 4081, 4156] avg=4096.0000 max=4161 max_rank=5 min=4041 min_rank=0 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[488, 515, 529, 504, 534, 533, 575, 528, 0, 0, 0, 0, 0, 0, 0, 0, 488, 546, 523, 538, 536, 538, 495, 489, 0, 0, 0, 0, 0, 0, 0, 0, 490, 550, 518, 537, 529, 540, 471, 495, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 509, 570, 514, 510, 511, 559, 0, 0, 0, 0, 0, 0, 0, 0, 530, 507, 512, 488, 536, 520, 454, 488, 13, 12, 0, 0, 0, 0, 0, 0, 452, 485, 481, 490, 486, 464, 524, 507, 6, 6, 0, 0, 0, 0, 0, 0, 467, 506, 486, 518, 528, 523, 532, 532, 7, 2, 0, 0, 0, 0, 0, 0, 511, 518, 495, 494, 502, 506, 487, 478, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4206, 4153, 4130, 4196, 4060, 3901, 4101, 4021] avg=4096.0000 max=4206 max_rank=0 min=3901 min_rank=5 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16083 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[542, 569, 505, 460, 486, 534, 489, 534, 0, 0, 0, 0, 0, 0, 0, 0, 545, 551, 538, 451, 512, 526, 510, 436, 0, 0, 0, 0, 0, 0, 0, 0, 522, 513, 524, 505, 541, 520, 532, 534, 0, 0, 0, 0, 0, 0, 0, 0, 498, 513, 541, 451, 509, 526, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 544, 500, 521, 515, 523, 504, 503, 11, 12, 0, 0, 0, 0, 0, 0, 492, 499, 547, 471, 504, 470, 499, 530, 6, 6, 0, 0, 0, 0, 0, 0, 468, 477, 509, 533, 513, 483, 530, 485, 6, 2, 0, 0, 0, 0, 0, 0, 506, 499, 516, 518, 528, 519, 516, 512, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4069, 4191, 4074, 4145, 4024, 4006, 4140] avg=4096.0000 max=4191 max_rank=2 min=4006 min_rank=6 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[522, 518, 510, 487, 478, 482, 522, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 519, 511, 512, 537, 523, 506, 486, 0, 0, 0, 0, 0, 0, 0, 0, 548, 531, 528, 503, 511, 494, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 518, 510, 506, 546, 511, 552, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 479, 517, 519, 499, 538, 494, 504, 505, 12, 12, 0, 0, 0, 0, 0, 0, 505, 508, 458, 483, 493, 504, 505, 504, 6, 6, 0, 0, 0, 0, 0, 0, 500, 530, 531, 542, 510, 470, 477, 567, 7, 2, 0, 0, 0, 0, 0, 0, 480, 491, 522, 501, 554, 531, 506, 505, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4121, 4148, 4160, 4079, 3972, 4136, 4120] avg=4096.0000 max=4160 max_rank=3 min=3972 min_rank=5 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 529, 555, 503, 518, 493, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0, 484, 537, 501, 516, 478, 530, 529, 484, 0, 0, 0, 0, 0, 0, 0, 0, 469, 524, 496, 537, 516, 529, 509, 507, 0, 0, 0, 0, 0, 0, 0, 0, 515, 511, 500, 512, 525, 496, 495, 462, 0, 0, 0, 0, 0, 0, 0, 0, 522, 514, 505, 523, 520, 489, 490, 491, 17, 17, 0, 0, 0, 0, 0, 0, 502, 548, 543, 541, 519, 514, 526, 511, 11, 10, 0, 0, 0, 0, 0, 0, 508, 522, 514, 498, 527, 533, 491, 503, 11, 9, 0, 0, 0, 0, 0, 0, 517, 461, 473, 527, 483, 485, 519, 504, 35, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4173, 4059, 4087, 4016, 4088, 4225, 4116, 4004] avg=4096.0000 max=4225 max_rank=5 min=4004 min_rank=7 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[506, 471, 515, 514, 519, 479, 564, 515, 0, 0, 0, 0, 0, 0, 0, 0, 525, 522, 504, 504, 494, 489, 535, 516, 0, 0, 0, 0, 0, 0, 0, 0, 474, 546, 486, 530, 471, 498, 528, 527, 0, 0, 0, 0, 0, 0, 0, 0, 489, 521, 522, 506, 492, 522, 502, 549, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 514, 517, 500, 522, 498, 530, 13, 12, 0, 0, 0, 0, 0, 0, 543, 508, 511, 519, 521, 490, 460, 522, 5, 5, 0, 0, 0, 0, 0, 0, 456, 524, 507, 535, 526, 521, 479, 539, 7, 2, 0, 0, 0, 0, 0, 0, 545, 501, 498, 516, 541, 516, 494, 490, 31, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4089, 4060, 4103, 4121, 4084, 4096, 4132] avg=4096.0000 max=4132 max_rank=7 min=4060 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 4224, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[496, 506, 487, 521, 499, 506, 521, 516, 474, 514, 501, 519, 501, 542, 518, 483, 532, 504, 479, 551, 515, 463, 505, 522, 523, 515, 541, 561, 494, 541, 529, 512, 510, 476, 494, 528, 506, 537, 494, 480, 502, 525, 493, 514, 500, 516, 546, 476, 540, 511, 508, 525, 496, 532, 525, 538, 521, 533, 516, 502, 529, 475, 540, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[528, 504, 496, 481, 499, 510, 470, 512, 530, 545, 502, 515, 535, 529, 510, 481, 469, 499, 542, 537, 535, 498, 488, 533, 513, 515, 491, 561, 491, 518, 508, 528, 545, 491, 477, 529, 514, 556, 508, 517, 494, 523, 495, 526, 523, 505, 484, 500, 529, 495, 520, 523, 496, 502, 517, 525, 499, 550, 510, 530, 484, 508, 524, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[498, 481, 507, 536, 489, 516, 484, 491, 539, 507, 525, 514, 500, 518, 513, 496, 480, 497, 511, 519, 521, 520, 503, 534, 497, 567, 501, 541, 513, 483, 516, 502, 517, 517, 516, 537, 560, 538, 484, 518, 514, 485, 513, 504, 485, 490, 548, 503, 527, 525, 527, 472, 522, 528, 508, 524, 564, 514, 479, 520, 525, 487, 498, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[519, 515, 486, 518, 528, 536, 469, 532, 500, 479, 536, 542, 504, 539, 532, 494, 513, 490, 523, 519, 523, 515, 543, 543, 498, 511, 499, 504, 497, 514, 494, 529, 504, 525, 524, 499, 500, 533, 511, 529, 509, 546, 501, 530, 501, 520, 492, 496, 536, 503, 532, 485, 509, 496, 515, 498, 503, 504, 512, 489, 506, 524, 467, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[495, 549, 541, 516, 481, 533, 524, 534, 486, 485, 520, 528, 479, 493, 534, 504, 496, 502, 554, 533, 523, 518, 498, 513, 496, 537, 475, 508, 524, 488, 547, 493, 498, 503, 513, 509, 526, 485, 528, 536, 506, 535, 483, 494, 505, 499, 530, 540, 519, 510, 488, 517, 494, 513, 510, 476, 525, 525, 520, 519, 506, 504, 541, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[472, 508, 511, 545, 525, 499, 485, 542, 488, 512, 558, 519, 498, 544, 505, 537, 512, 510, 492, 506, 465, 507, 485, 523, 478, 479, 497, 542, 485, 481, 530, 492, 552, 519, 532, 532, 477, 521, 539, 484, 512, 525, 494, 504, 492, 531, 530, 495, 533, 504, 542, 523, 486, 497, 545, 515, 530, 509, 543, 545, 514, 512, 489, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[503, 516, 504, 529, 498, 501, 554, 513, 501, 533, 525, 496, 566, 489, 505, 501, 520, 525, 527, 532, 457, 501, 515, 481, 498, 524, 527, 554, 501, 531, 484, 504, 523, 490, 500, 494, 514, 524, 519, 531, 490, 552, 508, 500, 528, 501, 510, 524, 541, 499, 526, 493, 479, 542, 493, 499, 522, 507, 558, 481, 514, 478, 496, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[496, 547, 492, 524, 455, 518, 528, 492, 485, 537, 516, 540, 534, 498, 506, 515, 512, 501, 546, 530, 495, 536, 502, 540, 459, 506, 493, 475, 528, 530, 497, 509, 515, 516, 525, 476, 521, 527, 489, 486, 487, 493, 523, 556, 553, 523, 483, 513, 492, 501, 550, 496, 498, 536, 491, 523, 547, 527, 501, 530, 534, 516, 508, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[528, 504, 496, 481, 499, 510, 470, 512, 14, 8, 0, 0, 0, 0, 0, 0, 530, 545, 490, 515, 535, 529, 510, 481, 0, 0, 0, 0, 0, 0, 0, 0, 469, 499, 542, 536, 535, 498, 488, 533, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 491, 561, 491, 518, 508, 528, 6, 2, 1, 0, 0, 0, 0, 0, 545, 491, 477, 529, 514, 542, 508, 517, 0, 0, 0, 0, 0, 0, 0, 0, 494, 523, 495, 526, 523, 505, 484, 500, 6, 4, 0, 0, 0, 0, 0, 0, 523, 495, 520, 523, 496, 502, 517, 525, 0, 0, 0, 0, 0, 0, 0, 0, 491, 550, 510, 530, 484, 508, 524, 496, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4022, 4135, 4100, 4134, 4123, 4060, 4101, 4093] avg=4096.0000 max=4135 max_rank=1 min=4022 min_rank=0 imbalance_ratio=1.009521 max_load_reduction=28633 max_load_reduction_pct=0.873810 moved_to_echo=16377 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 506, 487, 521, 499, 506, 521, 516, 13, 8, 0, 0, 0, 0, 0, 0, 474, 514, 489, 519, 501, 542, 518, 483, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 479, 550, 515, 463, 505, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 515, 541, 561, 494, 541, 529, 512, 6, 2, 1, 0, 0, 0, 0, 0, 510, 476, 494, 528, 506, 524, 494, 480, 0, 0, 0, 0, 0, 0, 0, 0, 502, 525, 493, 514, 500, 516, 546, 476, 6, 4, 0, 0, 0, 0, 0, 0, 534, 511, 508, 525, 496, 532, 525, 538, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 516, 502, 529, 475, 540, 489, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4040, 4070, 4225, 4012, 4082, 4169, 4097] avg=4096.0000 max=4225 max_rank=3 min=4012 min_rank=4 imbalance_ratio=1.031494 max_load_reduction=28543 max_load_reduction_pct=0.871063 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 515, 486, 518, 528, 536, 469, 532, 13, 9, 0, 0, 0, 0, 0, 0, 500, 479, 523, 542, 504, 539, 532, 494, 0, 0, 0, 0, 0, 0, 0, 0, 513, 490, 523, 518, 523, 515, 543, 543, 0, 0, 0, 0, 0, 0, 0, 0, 498, 511, 499, 504, 497, 514, 494, 529, 6, 2, 1, 0, 0, 0, 0, 0, 504, 525, 524, 499, 500, 520, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 509, 546, 501, 530, 501, 520, 492, 496, 6, 4, 0, 0, 0, 0, 0, 0, 530, 503, 532, 485, 509, 496, 515, 498, 0, 0, 0, 0, 0, 0, 0, 0, 495, 504, 512, 489, 506, 524, 467, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4113, 4168, 4055, 4112, 4105, 4068, 4022] avg=4096.0000 max=4168 max_rank=2 min=4022 min_rank=7 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16307 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[495, 549, 541, 516, 481, 533, 524, 534, 12, 8, 0, 0, 0, 0, 0, 0, 486, 485, 508, 528, 479, 493, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 502, 554, 532, 523, 518, 498, 513, 0, 0, 0, 0, 0, 0, 0, 0, 496, 537, 475, 508, 524, 488, 547, 493, 6, 2, 1, 0, 0, 0, 0, 0, 498, 503, 513, 509, 526, 473, 528, 536, 0, 0, 0, 0, 0, 0, 0, 0, 506, 535, 483, 494, 505, 499, 530, 540, 6, 4, 0, 0, 0, 0, 0, 0, 513, 510, 488, 517, 494, 513, 510, 476, 0, 0, 0, 0, 0, 0, 0, 0, 517, 525, 520, 519, 506, 504, 541, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4017, 4136, 4077, 4086, 4102, 4021, 4136] avg=4096.0000 max=4193 max_rank=0 min=4017 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16345 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 481, 507, 536, 489, 516, 484, 491, 14, 8, 0, 0, 0, 0, 0, 0, 539, 507, 513, 514, 500, 518, 513, 496, 0, 0, 0, 0, 0, 0, 0, 0, 480, 497, 511, 518, 521, 520, 503, 534, 0, 0, 0, 0, 0, 0, 0, 0, 497, 567, 501, 541, 513, 483, 516, 502, 6, 2, 1, 0, 0, 0, 0, 0, 517, 517, 516, 537, 560, 524, 484, 518, 0, 0, 0, 0, 0, 0, 0, 0, 514, 485, 513, 504, 485, 490, 548, 503, 7, 4, 0, 0, 0, 0, 0, 0, 521, 525, 527, 472, 522, 528, 508, 524, 0, 0, 0, 0, 0, 0, 0, 0, 555, 514, 479, 520, 525, 487, 498, 500, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4024, 4100, 4084, 4129, 4173, 4053, 4127, 4078] avg=4096.0000 max=4173 max_rank=4 min=4024 min_rank=0 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[496, 547, 492, 524, 455, 518, 528, 492, 13, 8, 0, 0, 0, 0, 0, 0, 485, 537, 504, 540, 534, 498, 506, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 501, 546, 529, 495, 536, 502, 540, 0, 0, 0, 0, 0, 0, 0, 0, 459, 506, 493, 475, 528, 530, 497, 509, 6, 2, 1, 0, 0, 0, 0, 0, 515, 516, 525, 476, 521, 514, 489, 486, 0, 0, 0, 0, 0, 0, 0, 0, 487, 493, 523, 556, 553, 523, 483, 513, 7, 4, 0, 0, 0, 0, 0, 0, 486, 501, 550, 496, 498, 536, 491, 523, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 501, 530, 534, 516, 508, 490, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4073, 4119, 4161, 4006, 4042, 4142, 4081, 4144] avg=4096.0000 max=4161 max_rank=2 min=4006 min_rank=3 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 516, 504, 529, 498, 501, 554, 513, 13, 8, 0, 0, 0, 0, 0, 0, 501, 533, 513, 496, 566, 489, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 527, 531, 457, 501, 515, 481, 0, 0, 0, 0, 0, 0, 0, 0, 498, 524, 527, 554, 501, 531, 484, 504, 6, 2, 1, 0, 0, 0, 0, 0, 523, 490, 500, 494, 514, 511, 519, 531, 0, 0, 0, 0, 0, 0, 0, 0, 490, 552, 508, 500, 528, 501, 510, 524, 6, 4, 0, 0, 0, 0, 0, 0, 535, 499, 526, 493, 479, 542, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0, 514, 507, 558, 481, 514, 478, 496, 517, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4104, 4057, 4132, 4082, 4123, 4066, 4065] avg=4096.0000 max=4139 max_rank=0 min=4057 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[472, 508, 511, 545, 525, 499, 485, 542, 18, 14, 0, 0, 0, 0, 0, 0, 488, 512, 537, 519, 498, 544, 505, 537, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 492, 498, 465, 507, 485, 523, 0, 0, 0, 0, 0, 0, 0, 0, 478, 479, 497, 542, 485, 481, 530, 492, 10, 8, 8, 0, 0, 0, 0, 0, 552, 519, 532, 532, 477, 503, 539, 484, 0, 0, 0, 0, 0, 0, 0, 0, 512, 525, 494, 504, 492, 531, 530, 495, 11, 7, 0, 0, 0, 0, 0, 0, 523, 504, 542, 523, 486, 497, 545, 515, 0, 0, 0, 0, 0, 0, 0, 0, 511, 509, 543, 545, 514, 512, 489, 510, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4140, 3992, 4010, 4138, 4101, 4135, 4133] avg=4096.0000 max=4140 max_rank=1 min=3992 min_rank=2 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16507 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[3968, 4224, 4096, 4352, 4096, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[511, 495, 538, 533, 526, 528, 516, 520, 512, 501, 535, 503, 530, 569, 515, 502, 534, 522, 522, 503, 483, 496, 514, 509, 497, 493, 512, 524, 532, 505, 475, 531, 499, 483, 534, 491, 522, 476, 542, 522, 508, 506, 487, 507, 525, 512, 514, 510, 496, 524, 499, 491, 505, 536, 524, 518, 545, 511, 504, 478, 509, 503, 497, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[520, 517, 554, 508, 510, 512, 533, 486, 521, 517, 517, 514, 507, 510, 523, 536, 538, 541, 525, 508, 500, 502, 518, 485, 502, 512, 510, 513, 497, 485, 508, 520, 485, 539, 475, 522, 506, 496, 520, 499, 509, 517, 542, 486, 510, 537, 517, 517, 507, 514, 483, 528, 466, 503, 486, 546, 521, 502, 544, 482, 526, 540, 495, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[520, 472, 545, 540, 534, 502, 490, 482, 487, 513, 500, 512, 518, 486, 497, 512, 497, 485, 516, 506, 529, 501, 520, 514, 520, 559, 503, 522, 483, 501, 537, 516, 496, 535, 489, 499, 530, 548, 525, 513, 520, 545, 493, 514, 443, 524, 517, 534, 516, 502, 523, 505, 502, 542, 504, 527, 528, 495, 543, 477, 503, 481, 541, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[519, 531, 516, 551, 513, 523, 507, 499, 546, 496, 491, 505, 477, 533, 504, 506, 485, 526, 566, 512, 519, 489, 530, 475, 492, 498, 516, 548, 502, 535, 544, 526, 489, 511, 543, 529, 515, 496, 515, 501, 506, 503, 508, 484, 489, 507, 493, 520, 478, 502, 529, 509, 545, 531, 483, 524, 486, 493, 490, 540, 537, 527, 487, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[556, 517, 535, 534, 512, 531, 509, 494, 505, 555, 517, 508, 534, 530, 538, 478, 482, 510, 559, 555, 517, 522, 535, 530, 530, 503, 488, 493, 534, 499, 488, 548, 503, 532, 498, 467, 504, 513, 517, 496, 524, 492, 508, 494, 524, 521, 501, 477, 491, 503, 519, 497, 514, 477, 493, 490, 546, 483, 473, 525, 531, 502, 521, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[573, 522, 543, 534, 537, 495, 498, 546, 538, 540, 537, 530, 506, 510, 520, 504, 539, 518, 517, 503, 492, 516, 493, 515, 495, 519, 490, 482, 516, 500, 499, 504, 486, 504, 514, 533, 500, 474, 501, 526, 513, 500, 512, 510, 495, 532, 512, 488, 512, 507, 497, 469, 538, 518, 509, 485, 517, 501, 513, 498, 517, 534, 484, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[548, 504, 502, 492, 496, 516, 521, 483, 499, 516, 525, 511, 555, 506, 520, 506, 545, 546, 488, 492, 512, 471, 553, 500, 504, 529, 522, 523, 509, 539, 489, 542, 529, 557, 527, 480, 516, 472, 519, 530, 513, 486, 484, 540, 483, 492, 509, 537, 499, 480, 507, 494, 533, 516, 493, 523, 481, 493, 508, 538, 512, 523, 529, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[519, 543, 554, 492, 545, 509, 497, 513, 525, 546, 495, 487, 509, 513, 503, 473, 538, 548, 498, 518, 517, 518, 554, 490, 526, 518, 495, 512, 539, 512, 488, 509, 547, 523, 496, 484, 533, 557, 510, 533, 500, 491, 502, 500, 501, 483, 482, 519, 491, 521, 486, 501, 530, 491, 526, 516, 481, 522, 529, 506, 493, 523, 468, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[511, 495, 482, 533, 526, 528, 516, 520, 0, 0, 0, 0, 0, 0, 0, 0, 512, 482, 535, 503, 530, 569, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 534, 522, 522, 503, 483, 496, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 497, 493, 512, 524, 532, 505, 475, 531, 0, 0, 0, 0, 0, 0, 0, 0, 499, 483, 534, 491, 522, 476, 542, 522, 5, 0, 0, 0, 0, 0, 0, 0, 508, 506, 487, 507, 525, 512, 514, 510, 42, 0, 0, 0, 0, 0, 0, 0, 496, 524, 499, 491, 505, 536, 524, 518, 21, 14, 0, 0, 0, 0, 0, 0, 545, 511, 504, 478, 509, 503, 497, 504, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4148, 4061, 4069, 4074, 4111, 4128, 4066] avg=4096.0000 max=4148 max_rank=1 min=4061 min_rank=2 imbalance_ratio=1.012695 max_load_reduction=28620 max_load_reduction_pct=0.873413 moved_to_echo=16379 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 517, 497, 508, 510, 512, 533, 486, 0, 0, 0, 0, 0, 0, 0, 0, 521, 498, 517, 514, 507, 510, 523, 536, 0, 0, 0, 0, 0, 0, 0, 0, 538, 541, 525, 508, 500, 502, 495, 485, 0, 0, 0, 0, 0, 0, 0, 0, 502, 512, 510, 513, 497, 485, 508, 520, 0, 0, 0, 0, 0, 0, 0, 0, 485, 539, 475, 522, 506, 496, 520, 499, 5, 0, 0, 0, 0, 0, 0, 0, 509, 517, 542, 486, 510, 537, 517, 517, 43, 0, 0, 0, 0, 0, 0, 0, 507, 514, 483, 528, 466, 503, 486, 546, 22, 14, 0, 0, 0, 0, 0, 0, 521, 502, 544, 482, 526, 540, 495, 499, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4126, 4094, 4047, 4047, 4178, 4069, 4124] avg=4096.0000 max=4178 max_rank=5 min=4047 min_rank=3 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[556, 517, 479, 534, 512, 531, 509, 494, 0, 0, 0, 0, 0, 0, 0, 0, 505, 535, 517, 508, 534, 530, 538, 478, 0, 0, 0, 0, 0, 0, 0, 0, 482, 510, 559, 555, 517, 522, 512, 530, 0, 0, 0, 0, 0, 0, 0, 0, 530, 503, 488, 493, 534, 499, 488, 548, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 498, 467, 504, 513, 517, 496, 5, 0, 0, 0, 0, 0, 0, 0, 524, 492, 508, 494, 524, 521, 501, 477, 42, 0, 0, 0, 0, 0, 0, 0, 491, 503, 519, 497, 514, 477, 493, 490, 22, 14, 0, 0, 0, 0, 0, 0, 546, 483, 473, 525, 531, 502, 521, 486, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4145, 4187, 4083, 4035, 4083, 4020, 4083] avg=4096.0000 max=4187 max_rank=2 min=4020 min_rank=6 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16221 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 472, 488, 540, 534, 502, 490, 482, 0, 0, 0, 0, 0, 0, 0, 0, 487, 494, 500, 512, 518, 486, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 497, 485, 516, 506, 529, 501, 497, 514, 0, 0, 0, 0, 0, 0, 0, 0, 520, 559, 503, 522, 483, 501, 537, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 489, 499, 530, 548, 525, 513, 5, 0, 0, 0, 0, 0, 0, 0, 520, 545, 493, 514, 443, 524, 517, 534, 43, 0, 0, 0, 0, 0, 0, 0, 516, 502, 523, 505, 502, 542, 504, 527, 22, 14, 0, 0, 0, 0, 0, 0, 528, 495, 543, 477, 503, 481, 541, 535, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4006, 4045, 4141, 4140, 4133, 4157, 4118] avg=4096.0000 max=4157 max_rank=6 min=4006 min_rank=1 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16548 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 531, 463, 551, 513, 523, 507, 499, 0, 0, 0, 0, 0, 0, 0, 0, 546, 479, 491, 505, 477, 533, 504, 506, 0, 0, 0, 0, 0, 0, 0, 0, 485, 526, 566, 512, 519, 489, 507, 475, 0, 0, 0, 0, 0, 0, 0, 0, 492, 498, 516, 548, 502, 535, 544, 526, 0, 0, 0, 0, 0, 0, 0, 0, 489, 511, 543, 529, 515, 496, 515, 501, 4, 0, 0, 0, 0, 0, 0, 0, 506, 503, 508, 484, 489, 507, 493, 520, 40, 0, 0, 0, 0, 0, 0, 0, 478, 502, 529, 509, 545, 531, 483, 524, 22, 13, 0, 0, 0, 0, 0, 0, 486, 493, 490, 540, 537, 527, 487, 518, 13, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4041, 4079, 4161, 4103, 4050, 4136, 4092] avg=4096.0000 max=4161 max_rank=3 min=4041 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[548, 504, 450, 492, 496, 516, 521, 483, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 525, 511, 555, 506, 520, 506, 0, 0, 0, 0, 0, 0, 0, 0, 545, 546, 488, 492, 512, 471, 529, 500, 0, 0, 0, 0, 0, 0, 0, 0, 504, 529, 522, 523, 509, 539, 489, 542, 0, 0, 0, 0, 0, 0, 0, 0, 529, 557, 527, 480, 516, 472, 519, 530, 5, 0, 0, 0, 0, 0, 0, 0, 513, 486, 484, 540, 483, 492, 509, 537, 39, 0, 0, 0, 0, 0, 0, 0, 499, 480, 507, 494, 533, 516, 493, 523, 23, 13, 0, 0, 0, 0, 0, 0, 481, 493, 508, 538, 512, 523, 529, 501, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4119, 4083, 4157, 4135, 4083, 4081, 4100] avg=4096.0000 max=4157 max_rank=3 min=4010 min_rank=0 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16399 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[573, 522, 478, 534, 537, 495, 498, 546, 0, 0, 0, 0, 0, 0, 0, 0, 538, 507, 537, 530, 506, 510, 520, 504, 0, 0, 0, 0, 0, 0, 0, 0, 539, 518, 517, 503, 492, 516, 462, 515, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 490, 482, 516, 500, 499, 504, 6, 0, 0, 0, 0, 0, 0, 0, 486, 504, 514, 533, 500, 474, 501, 526, 8, 0, 0, 0, 0, 0, 0, 0, 513, 500, 512, 510, 495, 532, 512, 488, 47, 0, 0, 0, 0, 0, 0, 0, 512, 507, 497, 469, 538, 518, 509, 485, 25, 18, 0, 0, 0, 0, 0, 0, 517, 501, 513, 498, 517, 534, 484, 538, 19, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4183, 4152, 4062, 4011, 4046, 4109, 4078, 4127] avg=4096.0000 max=4183 max_rank=0 min=4011 min_rank=3 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16360 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[519, 543, 497, 492, 545, 509, 497, 513, 0, 0, 0, 0, 0, 0, 0, 0, 525, 526, 495, 487, 509, 513, 503, 473, 0, 0, 0, 0, 0, 0, 0, 0, 538, 548, 498, 518, 517, 518, 530, 490, 0, 0, 0, 0, 0, 0, 0, 0, 526, 518, 495, 512, 539, 512, 488, 509, 0, 0, 0, 0, 0, 0, 0, 0, 547, 523, 496, 484, 533, 557, 510, 533, 5, 0, 0, 0, 0, 0, 0, 0, 500, 491, 502, 500, 501, 483, 482, 519, 43, 0, 0, 0, 0, 0, 0, 0, 491, 521, 486, 501, 530, 491, 526, 516, 23, 14, 0, 0, 0, 0, 0, 0, 481, 522, 529, 506, 493, 523, 468, 520, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4031, 4157, 4099, 4188, 4021, 4099, 4058] avg=4096.0000 max=4188 max_rank=4 min=4021 min_rank=5 imbalance_ratio=1.022461 max_load_reduction=28580 max_load_reduction_pct=0.872192 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4352, 4224, 3840, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[563, 483, 506, 489, 544, 516, 521, 515, 546, 548, 528, 510, 492, 486, 541, 516, 496, 501, 538, 470, 479, 503, 509, 480, 473, 493, 507, 556, 528, 529, 505, 503, 507, 524, 526, 553, 484, 521, 567, 487, 524, 523, 506, 520, 501, 506, 492, 518, 509, 530, 481, 544, 514, 499, 513, 513, 539, 504, 497, 487, 497, 513, 497, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[549, 553, 474, 539, 486, 511, 492, 505, 473, 481, 545, 510, 563, 497, 516, 530, 502, 522, 539, 493, 516, 486, 509, 538, 554, 504, 536, 524, 482, 526, 519, 503, 504, 525, 546, 521, 533, 542, 498, 565, 468, 512, 499, 498, 487, 482, 519, 472, 507, 543, 482, 482, 538, 528, 548, 539, 504, 491, 504, 491, 483, 496, 508, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[546, 558, 510, 509, 516, 504, 499, 505, 542, 535, 503, 531, 513, 501, 506, 489, 573, 502, 528, 501, 489, 520, 493, 513, 510, 499, 511, 518, 535, 501, 521, 483, 507, 506, 525, 515, 492, 501, 481, 502, 530, 537, 526, 502, 522, 471, 507, 501, 521, 530, 506, 529, 477, 500, 469, 548, 489, 504, 468, 539, 538, 516, 512, 533] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[503, 544, 507, 540, 525, 506, 538, 522, 536, 508, 560, 489, 506, 504, 500, 512, 493, 486, 500, 537, 538, 522, 482, 551, 508, 468, 529, 529, 539, 482, 488, 515, 493, 497, 486, 537, 501, 503, 503, 510, 495, 537, 533, 456, 544, 494, 525, 549, 521, 488, 509, 546, 508, 521, 554, 506, 494, 477, 504, 512, 484, 496, 487, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[515, 514, 546, 513, 535, 509, 542, 522, 531, 504, 528, 488, 513, 473, 516, 542, 497, 539, 519, 507, 523, 523, 520, 499, 503, 497, 488, 503, 525, 491, 506, 489, 522, 537, 528, 472, 524, 527, 497, 501, 499, 497, 518, 504, 558, 524, 498, 504, 515, 529, 533, 504, 543, 493, 516, 503, 500, 502, 507, 476, 482, 522, 504, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[556, 530, 485, 529, 577, 526, 529, 515, 503, 514, 499, 477, 516, 509, 467, 501, 489, 487, 474, 476, 531, 527, 483, 532, 546, 515, 507, 516, 519, 492, 522, 525, 540, 552, 466, 518, 523, 498, 546, 470, 502, 524, 479, 480, 497, 511, 526, 497, 525, 509, 508, 540, 522, 534, 508, 526, 517, 521, 499, 501, 527, 497, 517, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[525, 524, 544, 525, 548, 522, 505, 498, 494, 525, 487, 521, 507, 508, 507, 549, 542, 507, 496, 535, 541, 512, 505, 557, 518, 527, 510, 510, 461, 513, 511, 513, 513, 480, 494, 464, 555, 515, 492, 547, 516, 525, 519, 511, 521, 476, 491, 515, 539, 485, 490, 522, 487, 531, 578, 453, 494, 483, 513, 530, 508, 492, 475, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[503, 500, 492, 525, 490, 509, 494, 495, 540, 520, 535, 521, 504, 517, 514, 488, 479, 506, 547, 498, 513, 519, 511, 505, 531, 498, 500, 532, 516, 518, 499, 489, 534, 527, 525, 554, 485, 493, 505, 492, 509, 506, 555, 473, 512, 534, 477, 497, 527, 527, 522, 519, 537, 531, 509, 488, 536, 529, 529, 501, 515, 486, 529, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[504, 483, 506, 489, 544, 516, 521, 515, 0, 0, 0, 0, 0, 0, 0, 0, 546, 548, 521, 510, 492, 486, 541, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 538, 470, 479, 503, 509, 480, 7, 0, 0, 0, 0, 0, 0, 0, 473, 493, 507, 556, 528, 529, 505, 503, 11, 0, 0, 0, 0, 0, 0, 0, 507, 513, 526, 553, 484, 521, 567, 487, 0, 0, 0, 0, 0, 0, 0, 0, 524, 523, 506, 520, 501, 506, 492, 518, 31, 0, 0, 0, 0, 0, 0, 0, 509, 530, 481, 544, 514, 499, 479, 513, 0, 0, 0, 0, 0, 0, 0, 0, 539, 504, 497, 487, 497, 513, 497, 498, 59, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4078, 4160, 3983, 4105, 4158, 4121, 4069, 4094] avg=4096.0000 max=4160 max_rank=1 min=3983 min_rank=2 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16442 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[489, 558, 510, 509, 516, 504, 499, 505, 0, 0, 0, 0, 0, 0, 0, 0, 542, 535, 496, 531, 513, 501, 506, 489, 0, 0, 0, 0, 0, 0, 0, 0, 573, 502, 528, 501, 489, 520, 493, 513, 7, 0, 0, 0, 0, 0, 0, 0, 510, 499, 511, 518, 535, 501, 521, 483, 10, 0, 0, 0, 0, 0, 0, 0, 507, 496, 525, 515, 492, 501, 481, 502, 0, 0, 0, 0, 0, 0, 0, 0, 530, 537, 526, 502, 522, 471, 507, 501, 28, 0, 0, 0, 0, 0, 0, 0, 521, 530, 506, 529, 477, 500, 438, 548, 0, 0, 0, 0, 0, 0, 0, 0, 489, 504, 468, 539, 538, 516, 512, 533, 57, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4090, 4113, 4126, 4088, 4019, 4124, 4049, 4159] avg=4096.0000 max=4159 max_rank=7 min=4019 min_rank=4 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16351 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[491, 553, 474, 539, 486, 511, 492, 505, 0, 0, 0, 0, 0, 0, 0, 0, 473, 481, 538, 510, 563, 497, 516, 530, 0, 0, 0, 0, 0, 0, 0, 0, 502, 522, 539, 493, 516, 486, 509, 538, 7, 0, 0, 0, 0, 0, 0, 0, 554, 504, 536, 524, 482, 526, 519, 503, 11, 0, 0, 0, 0, 0, 0, 0, 504, 514, 546, 521, 533, 542, 498, 565, 0, 0, 0, 0, 0, 0, 0, 0, 468, 512, 499, 498, 487, 482, 519, 472, 33, 0, 0, 0, 0, 0, 0, 0, 507, 543, 482, 482, 538, 528, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 504, 491, 504, 491, 483, 496, 508, 476, 58, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4051, 4108, 4112, 4159, 4223, 3970, 4131, 4014] avg=4096.0000 max=4223 max_rank=4 min=3970 min_rank=5 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[450, 544, 507, 540, 525, 506, 538, 522, 0, 0, 0, 0, 0, 0, 0, 0, 536, 508, 552, 489, 506, 504, 500, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 500, 537, 538, 522, 482, 551, 8, 0, 0, 0, 0, 0, 0, 0, 508, 468, 529, 529, 539, 482, 488, 515, 10, 0, 0, 0, 0, 0, 0, 0, 493, 487, 486, 537, 501, 503, 503, 510, 0, 0, 0, 0, 0, 0, 0, 0, 495, 537, 533, 456, 544, 494, 525, 549, 33, 0, 0, 0, 0, 0, 0, 0, 521, 488, 509, 546, 508, 521, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 494, 477, 504, 512, 484, 496, 487, 531, 53, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4107, 4117, 4068, 4020, 4166, 4117, 4041] avg=4096.0000 max=4166 max_rank=5 min=4020 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16344 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[461, 514, 546, 513, 535, 509, 542, 522, 0, 0, 0, 0, 0, 0, 0, 0, 531, 504, 521, 488, 513, 473, 516, 542, 0, 0, 0, 0, 0, 0, 0, 0, 497, 539, 519, 507, 523, 523, 520, 499, 7, 0, 0, 0, 0, 0, 0, 0, 503, 497, 488, 503, 525, 491, 506, 489, 11, 0, 0, 0, 0, 0, 0, 0, 522, 526, 528, 472, 524, 527, 497, 501, 0, 0, 0, 0, 0, 0, 0, 0, 499, 497, 518, 504, 558, 524, 498, 504, 31, 0, 0, 0, 0, 0, 0, 0, 515, 529, 533, 504, 543, 493, 482, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 502, 507, 476, 482, 522, 504, 509, 54, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4088, 4134, 4013, 4097, 4133, 4102, 4059] avg=4096.0000 max=4142 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[470, 524, 544, 525, 548, 522, 505, 498, 0, 0, 0, 0, 0, 0, 0, 0, 494, 525, 481, 521, 507, 508, 507, 549, 0, 0, 0, 0, 0, 0, 0, 0, 542, 507, 496, 535, 541, 512, 505, 557, 6, 0, 0, 0, 0, 0, 0, 0, 518, 527, 510, 510, 461, 513, 511, 513, 10, 0, 0, 0, 0, 0, 0, 0, 513, 470, 494, 464, 555, 515, 492, 547, 0, 0, 0, 0, 0, 0, 0, 0, 516, 525, 519, 511, 521, 476, 491, 515, 35, 0, 0, 0, 0, 0, 0, 0, 539, 485, 490, 522, 487, 531, 540, 453, 0, 0, 0, 0, 0, 0, 0, 0, 494, 483, 513, 530, 508, 492, 475, 507, 55, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4136, 4092, 4201, 4073, 4050, 4109, 4047, 4060] avg=4096.0000 max=4201 max_rank=2 min=4047 min_rank=6 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16266 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[498, 530, 485, 529, 577, 526, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 492, 477, 516, 509, 467, 501, 0, 0, 0, 0, 0, 0, 0, 0, 489, 487, 474, 476, 531, 527, 483, 532, 7, 0, 0, 0, 0, 0, 0, 0, 546, 515, 507, 516, 519, 492, 522, 525, 11, 0, 0, 0, 0, 0, 0, 0, 540, 541, 466, 518, 523, 498, 546, 470, 0, 0, 0, 0, 0, 0, 0, 0, 502, 524, 479, 480, 497, 511, 526, 497, 31, 0, 0, 0, 0, 0, 0, 0, 525, 509, 508, 540, 522, 534, 474, 526, 0, 0, 0, 0, 0, 0, 0, 0, 517, 521, 499, 501, 527, 497, 517, 514, 58, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4189, 3979, 4006, 4153, 4102, 4047, 4138, 4154] avg=4096.0000 max=4189 max_rank=0 min=3979 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[445, 500, 492, 525, 490, 509, 494, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 520, 517, 521, 504, 517, 514, 488, 0, 0, 0, 0, 0, 0, 0, 0, 479, 506, 547, 498, 513, 519, 511, 505, 11, 0, 0, 0, 0, 0, 0, 0, 531, 498, 500, 532, 516, 518, 499, 489, 16, 7, 3, 0, 0, 0, 0, 0, 534, 511, 525, 554, 485, 493, 505, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 506, 555, 473, 512, 534, 477, 497, 35, 0, 0, 0, 0, 0, 0, 0, 527, 527, 522, 519, 537, 531, 464, 488, 0, 0, 0, 0, 0, 0, 0, 0, 536, 529, 529, 501, 515, 486, 529, 497, 58, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3950, 4121, 4089, 4109, 4099, 4098, 4115, 4187] avg=4096.0000 max=4187 max_rank=7 min=3950 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3840, 4224, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 3968, 4224, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 14:57:43] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 280.4 | throughput per GPU (TFLOP/s/GPU): 440.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.606937E+00 | loss scale: 1.0 | grad norm: 3.895 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[492, 479, 554, 545, 510, 507, 503, 532, 529, 554, 515, 472, 507, 540, 502, 509, 533, 514, 521, 512, 514, 518, 535, 512, 532, 518, 504, 503, 512, 522, 488, 520, 495, 480, 528, 501, 517, 524, 515, 524, 541, 466, 529, 457, 505, 496, 525, 531, 504, 506, 525, 493, 508, 476, 488, 535, 508, 471, 505, 531, 550, 498, 515, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[481, 506, 515, 516, 517, 513, 540, 508, 514, 502, 531, 522, 487, 484, 496, 553, 501, 513, 558, 505, 517, 522, 579, 510, 501, 553, 513, 479, 533, 503, 514, 517, 525, 463, 536, 493, 517, 488, 518, 479, 490, 459, 490, 524, 516, 476, 495, 519, 501, 518, 521, 530, 526, 478, 513, 522, 528, 520, 519, 512, 532, 524, 508, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[544, 522, 509, 537, 495, 503, 534, 503, 502, 535, 535, 514, 484, 504, 484, 508, 505, 531, 472, 493, 498, 494, 538, 528, 510, 515, 534, 498, 525, 534, 500, 526, 517, 536, 512, 518, 491, 512, 513, 510, 521, 502, 505, 533, 480, 504, 541, 491, 515, 523, 511, 498, 513, 471, 509, 529, 486, 527, 512, 518, 525, 516, 499, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[522, 504, 521, 528, 516, 489, 537, 492, 494, 511, 526, 552, 528, 500, 497, 523, 519, 526, 559, 484, 493, 554, 480, 531, 529, 456, 509, 529, 543, 518, 537, 557, 493, 494, 539, 501, 508, 494, 513, 496, 527, 522, 469, 523, 515, 454, 503, 480, 503, 523, 494, 512, 520, 456, 536, 512, 527, 484, 512, 498, 544, 526, 503, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[477, 546, 501, 509, 510, 522, 537, 492, 499, 521, 504, 487, 503, 545, 522, 518, 525, 509, 552, 514, 528, 485, 498, 489, 509, 491, 526, 528, 514, 536, 493, 526, 493, 517, 520, 512, 522, 515, 503, 495, 483, 545, 511, 493, 497, 504, 514, 504, 501, 524, 534, 518, 508, 579, 480, 488, 488, 535, 489, 531, 535, 513, 483, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[508, 497, 516, 517, 557, 496, 466, 480, 520, 510, 516, 504, 534, 522, 487, 553, 481, 529, 482, 510, 507, 519, 527, 523, 513, 529, 517, 550, 436, 511, 516, 514, 527, 504, 500, 495, 536, 481, 539, 487, 501, 530, 510, 481, 546, 477, 584, 505, 512, 488, 520, 505, 487, 542, 543, 521, 524, 503, 493, 499, 500, 532, 508, 541] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[545, 514, 514, 510, 518, 542, 502, 527, 523, 545, 525, 501, 486, 455, 519, 483, 536, 539, 483, 522, 481, 538, 555, 506, 501, 503, 500, 511, 499, 534, 509, 502, 514, 474, 474, 521, 550, 499, 542, 528, 517, 488, 473, 530, 522, 517, 525, 484, 504, 482, 551, 503, 521, 518, 500, 544, 489, 512, 502, 503, 529, 512, 506, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[529, 516, 506, 510, 505, 477, 509, 556, 507, 490, 536, 530, 505, 526, 505, 500, 525, 519, 481, 557, 523, 517, 517, 497, 557, 514, 515, 499, 497, 487, 528, 504, 537, 519, 520, 515, 481, 525, 503, 527, 530, 485, 481, 501, 514, 510, 507, 509, 535, 501, 526, 495, 485, 506, 512, 491, 541, 530, 526, 477, 514, 514, 512, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[481, 506, 515, 502, 517, 513, 540, 508, 0, 0, 0, 0, 0, 0, 0, 0, 514, 502, 528, 522, 487, 484, 496, 553, 0, 0, 0, 0, 0, 0, 0, 0, 501, 513, 558, 505, 517, 522, 542, 510, 0, 0, 0, 0, 0, 0, 0, 0, 501, 553, 513, 479, 533, 503, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 525, 463, 536, 493, 517, 488, 518, 479, 14, 2, 0, 0, 0, 0, 0, 0, 490, 459, 490, 524, 516, 476, 495, 519, 37, 15, 0, 0, 0, 0, 0, 0, 501, 518, 521, 530, 526, 478, 513, 522, 6, 3, 0, 0, 0, 0, 0, 0, 528, 520, 519, 512, 524, 524, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4082, 4086, 4168, 4098, 4035, 4021, 4118, 4160] avg=4096.0000 max=4168 max_rank=2 min=4021 min_rank=5 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[544, 522, 509, 522, 495, 503, 534, 503, 0, 0, 0, 0, 0, 0, 0, 0, 502, 535, 532, 514, 484, 504, 484, 508, 0, 0, 0, 0, 0, 0, 0, 0, 505, 531, 472, 493, 498, 494, 504, 528, 0, 0, 0, 0, 0, 0, 0, 0, 510, 515, 534, 498, 525, 534, 500, 511, 0, 0, 0, 0, 0, 0, 0, 0, 517, 536, 512, 518, 491, 512, 513, 510, 15, 2, 0, 0, 0, 0, 0, 0, 521, 502, 505, 533, 480, 504, 541, 491, 34, 15, 0, 0, 0, 0, 0, 0, 515, 523, 511, 498, 513, 471, 509, 529, 5, 3, 0, 0, 0, 0, 0, 0, 486, 527, 512, 518, 518, 516, 499, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4063, 4025, 4127, 4126, 4126, 4077, 4092] avg=4096.0000 max=4132 max_rank=0 min=4025 min_rank=2 imbalance_ratio=1.008789 max_load_reduction=28636 max_load_reduction_pct=0.873901 moved_to_echo=16421 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[522, 504, 521, 514, 516, 489, 537, 492, 0, 0, 0, 0, 0, 0, 0, 0, 494, 511, 523, 552, 528, 500, 497, 523, 0, 0, 0, 0, 0, 0, 0, 0, 519, 526, 559, 484, 493, 554, 449, 531, 0, 0, 0, 0, 0, 0, 0, 0, 529, 456, 509, 529, 543, 518, 537, 541, 0, 0, 0, 0, 0, 0, 0, 0, 493, 494, 539, 501, 508, 494, 513, 496, 14, 2, 0, 0, 0, 0, 0, 0, 527, 522, 469, 523, 515, 454, 503, 480, 31, 16, 0, 0, 0, 0, 0, 0, 503, 523, 494, 512, 520, 456, 536, 512, 6, 3, 0, 0, 0, 0, 0, 0, 527, 484, 512, 498, 536, 526, 503, 523, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4095, 4128, 4115, 4162, 4054, 4040, 4065, 4109] avg=4096.0000 max=4162 max_rank=3 min=4040 min_rank=5 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[492, 479, 554, 530, 510, 507, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 529, 554, 512, 472, 507, 540, 502, 509, 0, 0, 0, 0, 0, 0, 0, 0, 533, 514, 521, 512, 514, 518, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 532, 518, 504, 503, 512, 522, 488, 505, 0, 0, 0, 0, 0, 0, 0, 0, 495, 480, 528, 501, 517, 524, 515, 524, 15, 2, 0, 0, 0, 0, 0, 0, 541, 466, 529, 457, 505, 496, 525, 531, 34, 15, 0, 0, 0, 0, 0, 0, 504, 506, 525, 493, 508, 476, 488, 535, 6, 3, 0, 0, 0, 0, 0, 0, 508, 471, 505, 531, 542, 498, 515, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4107, 4125, 4125, 4084, 4101, 4099, 4044, 4083] avg=4096.0000 max=4125 max_rank=1 min=4044 min_rank=6 imbalance_ratio=1.007080 max_load_reduction=28643 max_load_reduction_pct=0.874115 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[477, 546, 501, 495, 510, 522, 537, 492, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 501, 487, 503, 545, 522, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 552, 514, 528, 485, 466, 489, 0, 0, 0, 0, 0, 0, 0, 0, 509, 491, 526, 528, 514, 536, 493, 511, 0, 0, 0, 0, 0, 0, 0, 0, 493, 517, 520, 512, 522, 515, 503, 495, 14, 2, 0, 0, 0, 0, 0, 0, 483, 545, 511, 493, 497, 504, 514, 504, 32, 15, 0, 0, 0, 0, 0, 0, 501, 524, 534, 518, 508, 579, 480, 488, 6, 3, 0, 0, 0, 0, 0, 0, 488, 535, 489, 531, 527, 513, 483, 518, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4080, 4096, 4068, 4108, 4093, 4098, 4141, 4084] avg=4096.0000 max=4141 max_rank=6 min=4068 min_rank=2 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16416 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[508, 497, 516, 500, 557, 496, 466, 480, 0, 0, 0, 0, 0, 0, 0, 0, 520, 510, 510, 504, 534, 522, 487, 553, 0, 0, 0, 0, 0, 0, 0, 0, 481, 529, 482, 510, 507, 519, 489, 523, 0, 0, 0, 0, 0, 0, 0, 0, 513, 529, 517, 550, 436, 511, 516, 487, 0, 0, 0, 0, 0, 0, 0, 0, 527, 504, 500, 495, 536, 481, 539, 487, 17, 5, 7, 0, 0, 0, 0, 0, 501, 530, 510, 481, 546, 477, 584, 505, 38, 20, 0, 0, 0, 0, 0, 0, 512, 488, 520, 505, 487, 542, 543, 521, 8, 6, 0, 0, 0, 0, 0, 0, 524, 503, 493, 499, 487, 532, 508, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4140, 4040, 4059, 4098, 4192, 4132, 4087] avg=4096.0000 max=4192 max_rank=5 min=4020 min_rank=0 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[545, 514, 514, 496, 518, 542, 502, 527, 0, 0, 0, 0, 0, 0, 0, 0, 523, 545, 522, 501, 486, 455, 519, 483, 0, 0, 0, 0, 0, 0, 0, 0, 536, 539, 483, 522, 481, 538, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 503, 500, 511, 499, 534, 509, 487, 0, 0, 0, 0, 0, 0, 0, 0, 514, 474, 474, 521, 550, 499, 542, 528, 14, 2, 0, 0, 0, 0, 0, 0, 517, 488, 473, 530, 522, 517, 525, 484, 36, 15, 0, 0, 0, 0, 0, 0, 504, 482, 551, 503, 521, 518, 500, 544, 6, 3, 0, 0, 0, 0, 0, 0, 489, 512, 502, 503, 521, 512, 506, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4034, 4124, 4044, 4118, 4107, 4132, 4051] avg=4096.0000 max=4158 max_rank=0 min=4034 min_rank=1 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16408 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[529, 516, 506, 496, 505, 477, 509, 556, 0, 0, 0, 0, 0, 0, 0, 0, 507, 490, 533, 530, 505, 526, 505, 500, 0, 0, 0, 0, 0, 0, 0, 0, 525, 519, 481, 557, 523, 517, 484, 497, 0, 0, 0, 0, 0, 0, 0, 0, 557, 514, 515, 499, 497, 487, 528, 489, 0, 0, 0, 0, 0, 0, 0, 0, 537, 519, 520, 515, 481, 525, 503, 527, 14, 2, 0, 0, 0, 0, 0, 0, 530, 485, 481, 501, 514, 510, 507, 509, 33, 15, 0, 0, 0, 0, 0, 0, 535, 501, 526, 495, 485, 506, 512, 491, 5, 3, 0, 0, 0, 0, 0, 0, 541, 530, 526, 477, 507, 514, 512, 495, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4094, 4096, 4103, 4086, 4143, 4085, 4059, 4102] avg=4096.0000 max=4143 max_rank=4 min=4059 min_rank=6 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16389 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 3968, 4096, 4096, 3968, 4224, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[494, 498, 524, 530, 489, 523, 529, 502, 481, 492, 507, 534, 524, 488, 544, 522, 506, 508, 511, 521, 549, 517, 509, 536, 571, 510, 501, 528, 487, 520, 481, 518, 503, 523, 481, 507, 508, 514, 465, 528, 521, 536, 488, 540, 499, 494, 506, 505, 503, 512, 503, 503, 523, 488, 482, 528, 554, 499, 523, 518, 549, 497, 489, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[516, 479, 511, 532, 507, 502, 491, 499, 459, 517, 522, 561, 514, 551, 510, 478, 490, 514, 498, 509, 587, 490, 495, 574, 533, 508, 517, 525, 492, 514, 527, 517, 509, 516, 512, 518, 504, 506, 477, 515, 506, 506, 513, 494, 520, 512, 514, 526, 522, 515, 522, 482, 523, 532, 516, 525, 527, 522, 534, 520, 460, 470, 482, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[536, 512, 513, 518, 502, 478, 497, 516, 570, 515, 558, 542, 547, 491, 512, 505, 542, 489, 506, 488, 519, 544, 490, 493, 521, 515, 544, 523, 516, 521, 516, 500, 509, 490, 489, 495, 515, 491, 511, 526, 525, 496, 514, 529, 473, 510, 492, 519, 539, 531, 534, 496, 482, 557, 488, 503, 496, 501, 510, 510, 516, 519, 458, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[500, 475, 531, 484, 544, 546, 498, 526, 508, 507, 520, 501, 464, 503, 473, 571, 515, 523, 491, 529, 499, 510, 485, 504, 514, 531, 511, 541, 507, 510, 562, 505, 502, 559, 542, 510, 487, 516, 475, 513, 513, 521, 497, 535, 494, 503, 474, 502, 543, 506, 511, 520, 533, 502, 538, 519, 514, 505, 505, 521, 501, 539, 476, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[478, 500, 514, 527, 531, 508, 522, 550, 491, 505, 481, 559, 509, 564, 505, 548, 550, 520, 484, 520, 545, 545, 509, 527, 501, 505, 519, 503, 508, 500, 557, 501, 538, 489, 516, 511, 507, 493, 503, 542, 520, 527, 499, 506, 500, 520, 514, 504, 517, 494, 521, 475, 465, 482, 528, 519, 513, 545, 467, 496, 488, 496, 484, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[529, 541, 521, 507, 531, 528, 509, 487, 531, 555, 542, 503, 491, 500, 473, 511, 535, 544, 467, 518, 530, 505, 510, 489, 496, 495, 514, 511, 488, 523, 483, 545, 538, 527, 536, 513, 522, 493, 531, 471, 534, 475, 507, 499, 527, 524, 500, 545, 531, 474, 554, 473, 492, 533, 505, 490, 510, 529, 510, 520, 491, 496, 515, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[492, 529, 471, 520, 518, 508, 531, 514, 524, 470, 540, 570, 529, 560, 489, 521, 484, 535, 509, 511, 507, 489, 463, 552, 502, 537, 523, 545, 509, 492, 523, 525, 491, 505, 479, 534, 506, 506, 532, 532, 514, 503, 515, 518, 508, 505, 486, 508, 468, 490, 463, 498, 515, 499, 534, 531, 544, 486, 536, 520, 496, 519, 544, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[468, 484, 519, 493, 534, 515, 528, 541, 512, 487, 495, 541, 512, 517, 471, 504, 491, 515, 550, 513, 504, 512, 501, 539, 488, 508, 515, 472, 495, 508, 510, 509, 509, 503, 557, 515, 550, 496, 518, 520, 516, 488, 499, 516, 500, 522, 524, 517, 485, 490, 530, 498, 531, 514, 527, 548, 493, 491, 522, 489, 533, 576, 492, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[516, 479, 511, 532, 507, 502, 491, 499, 2, 0, 0, 0, 0, 0, 0, 0, 459, 517, 522, 531, 514, 551, 510, 478, 0, 0, 0, 0, 0, 0, 0, 0, 490, 514, 498, 509, 568, 490, 495, 574, 0, 0, 0, 0, 0, 0, 0, 0, 533, 508, 517, 525, 492, 514, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 512, 518, 504, 506, 477, 515, 8, 0, 0, 0, 0, 0, 0, 0, 506, 506, 513, 494, 520, 512, 514, 526, 10, 10, 0, 0, 0, 0, 0, 0, 522, 515, 522, 482, 523, 532, 516, 525, 9, 5, 0, 0, 0, 0, 0, 0, 527, 522, 534, 520, 460, 470, 482, 529, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4039, 4082, 4138, 4118, 4065, 4111, 4151, 4064] avg=4096.0000 max=4151 max_rank=6 min=4039 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[536, 512, 513, 518, 502, 478, 497, 516, 2, 0, 0, 0, 0, 0, 0, 0, 570, 515, 558, 514, 547, 491, 512, 505, 0, 0, 0, 0, 0, 0, 0, 0, 542, 489, 506, 488, 502, 544, 490, 493, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 544, 523, 516, 521, 501, 500, 0, 0, 0, 0, 0, 0, 0, 0, 509, 490, 489, 495, 515, 491, 511, 526, 8, 0, 0, 0, 0, 0, 0, 0, 525, 496, 514, 529, 473, 510, 492, 519, 9, 9, 0, 0, 0, 0, 0, 0, 539, 531, 534, 496, 482, 557, 488, 503, 8, 5, 0, 0, 0, 0, 0, 0, 496, 501, 510, 510, 516, 519, 458, 505, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4212, 4054, 4141, 4034, 4076, 4143, 4034] avg=4096.0000 max=4212 max_rank=1 min=4034 min_rank=4 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16287 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 498, 524, 530, 489, 523, 529, 502, 2, 0, 0, 0, 0, 0, 0, 0, 481, 492, 507, 506, 524, 488, 544, 522, 0, 0, 0, 0, 0, 0, 0, 0, 506, 508, 511, 521, 531, 517, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 571, 510, 501, 528, 487, 520, 468, 518, 0, 0, 0, 0, 0, 0, 0, 0, 503, 523, 481, 507, 508, 514, 465, 528, 7, 0, 0, 0, 0, 0, 0, 0, 521, 536, 488, 540, 499, 494, 506, 505, 9, 9, 0, 0, 0, 0, 0, 0, 503, 512, 503, 503, 523, 488, 482, 528, 9, 4, 0, 0, 0, 0, 0, 0, 554, 499, 523, 518, 549, 497, 489, 525, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4064, 4139, 4103, 4036, 4107, 4055, 4173] avg=4096.0000 max=4173 max_rank=7 min=4036 min_rank=4 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[500, 475, 531, 484, 544, 546, 498, 526, 2, 0, 0, 0, 0, 0, 0, 0, 508, 507, 520, 475, 464, 503, 473, 571, 0, 0, 0, 0, 0, 0, 0, 0, 515, 523, 491, 529, 483, 510, 485, 504, 0, 0, 0, 0, 0, 0, 0, 0, 514, 531, 511, 541, 507, 510, 546, 505, 0, 0, 0, 0, 0, 0, 0, 0, 502, 559, 542, 510, 487, 516, 475, 513, 9, 0, 0, 0, 0, 0, 0, 0, 513, 521, 497, 535, 494, 503, 474, 502, 8, 8, 0, 0, 0, 0, 0, 0, 543, 506, 511, 520, 533, 502, 538, 519, 8, 5, 0, 0, 0, 0, 0, 0, 514, 505, 505, 521, 501, 539, 476, 504, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4021, 4040, 4165, 4113, 4055, 4185, 4083] avg=4096.0000 max=4185 max_rank=6 min=4021 min_rank=1 imbalance_ratio=1.021729 max_load_reduction=28583 max_load_reduction_pct=0.872284 moved_to_echo=16436 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[468, 484, 519, 493, 534, 515, 528, 541, 2, 0, 0, 0, 0, 0, 0, 0, 512, 487, 495, 513, 512, 517, 471, 504, 0, 0, 0, 0, 0, 0, 0, 0, 491, 515, 550, 513, 488, 512, 501, 539, 0, 0, 0, 0, 0, 0, 0, 0, 488, 508, 515, 472, 495, 508, 495, 509, 0, 0, 0, 0, 0, 0, 0, 0, 509, 503, 557, 515, 550, 496, 518, 520, 8, 0, 0, 0, 0, 0, 0, 0, 516, 488, 499, 516, 500, 522, 524, 517, 9, 8, 0, 0, 0, 0, 0, 0, 485, 490, 530, 498, 531, 514, 527, 548, 8, 5, 0, 0, 0, 0, 0, 0, 493, 491, 522, 489, 533, 576, 492, 548, 19, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4084, 4011, 4109, 3990, 4176, 4099, 4136, 4163] avg=4096.0000 max=4176 max_rank=4 min=3990 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16574 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[478, 500, 514, 527, 531, 508, 522, 550, 2, 0, 0, 0, 0, 0, 0, 0, 491, 505, 481, 530, 509, 564, 505, 548, 0, 0, 0, 0, 0, 0, 0, 0, 550, 520, 484, 520, 527, 545, 509, 527, 0, 0, 0, 0, 0, 0, 0, 0, 501, 505, 519, 503, 508, 500, 541, 501, 0, 0, 0, 0, 0, 0, 0, 0, 538, 489, 516, 511, 507, 493, 503, 542, 9, 0, 0, 0, 0, 0, 0, 0, 520, 527, 499, 506, 500, 520, 514, 504, 9, 9, 0, 0, 0, 0, 0, 0, 517, 494, 521, 475, 465, 482, 528, 519, 9, 5, 0, 0, 0, 0, 0, 0, 513, 545, 467, 496, 488, 496, 484, 503, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4133, 4182, 4078, 4108, 4108, 4015, 4012] avg=4096.0000 max=4182 max_rank=2 min=4012 min_rank=7 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[529, 541, 521, 507, 531, 528, 509, 487, 4, 0, 0, 0, 0, 0, 0, 0, 531, 555, 542, 469, 491, 500, 473, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 544, 467, 518, 504, 505, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 514, 511, 488, 523, 460, 545, 0, 0, 0, 0, 0, 0, 0, 0, 538, 527, 536, 513, 522, 493, 531, 471, 12, 0, 0, 0, 0, 0, 0, 0, 534, 475, 507, 499, 527, 524, 500, 545, 13, 13, 0, 0, 0, 0, 0, 0, 531, 474, 554, 473, 492, 533, 505, 490, 13, 7, 0, 0, 0, 0, 0, 0, 510, 529, 510, 520, 491, 496, 515, 491, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4072, 4072, 4032, 4143, 4137, 4072, 4083] avg=4096.0000 max=4157 max_rank=0 min=4032 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 529, 471, 520, 518, 508, 531, 514, 2, 0, 0, 0, 0, 0, 0, 0, 524, 470, 540, 540, 529, 560, 489, 521, 0, 0, 0, 0, 0, 0, 0, 0, 484, 535, 509, 511, 491, 489, 463, 552, 0, 0, 0, 0, 0, 0, 0, 0, 502, 537, 523, 545, 509, 492, 508, 525, 0, 0, 0, 0, 0, 0, 0, 0, 491, 505, 479, 534, 506, 506, 532, 532, 8, 0, 0, 0, 0, 0, 0, 0, 514, 503, 515, 518, 508, 505, 486, 508, 10, 8, 0, 0, 0, 0, 0, 0, 468, 490, 463, 498, 515, 499, 534, 531, 8, 5, 0, 0, 0, 0, 0, 0, 544, 486, 536, 520, 496, 519, 544, 491, 20, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4173, 4034, 4141, 4093, 4075, 4011, 4156] avg=4096.0000 max=4173 max_rank=1 min=4011 min_rank=6 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16335 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 3968, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 3968, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[551, 471, 506, 533, 505, 494, 519, 526, 511, 560, 533, 496, 553, 538, 499, 530, 494, 506, 491, 524, 495, 503, 533, 534, 495, 527, 499, 540, 506, 522, 536, 510, 510, 514, 481, 488, 561, 494, 486, 514, 490, 510, 549, 517, 508, 510, 540, 521, 528, 475, 493, 496, 507, 503, 512, 495, 491, 479, 510, 542, 473, 516, 528, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[516, 516, 540, 523, 521, 502, 482, 524, 559, 532, 480, 490, 487, 509, 542, 521, 536, 506, 509, 511, 554, 494, 512, 521, 506, 491, 478, 512, 526, 491, 526, 501, 511, 508, 524, 461, 534, 567, 561, 494, 527, 507, 508, 487, 531, 495, 518, 475, 482, 518, 511, 519, 519, 521, 494, 516, 478, 506, 537, 536, 492, 501, 485, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[512, 505, 530, 486, 528, 523, 496, 484, 515, 521, 552, 475, 519, 526, 491, 502, 485, 524, 465, 479, 485, 503, 522, 521, 499, 493, 476, 506, 544, 471, 527, 525, 538, 494, 549, 509, 523, 537, 520, 526, 534, 517, 502, 489, 545, 534, 490, 513, 530, 524, 476, 501, 496, 497, 530, 568, 535, 510, 537, 492, 511, 502, 521, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 498, 516, 529, 509, 507, 477, 504, 519, 511, 532, 500, 473, 492, 537, 550, 541, 481, 532, 522, 524, 489, 538, 508, 508, 513, 510, 526, 548, 492, 501, 512, 544, 491, 543, 490, 502, 502, 502, 504, 496, 523, 506, 508, 531, 495, 503, 518, 498, 527, 519, 512, 508, 513, 502, 524, 496, 488, 482, 517, 543, 528, 516, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[506, 546, 509, 514, 534, 542, 511, 493, 537, 524, 533, 515, 525, 500, 513, 526, 483, 524, 508, 500, 529, 516, 519, 519, 479, 533, 519, 517, 526, 518, 514, 530, 531, 500, 492, 515, 480, 541, 529, 537, 502, 513, 494, 518, 487, 497, 501, 521, 509, 469, 519, 488, 506, 522, 525, 498, 506, 506, 508, 498, 505, 512, 496, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[498, 542, 477, 510, 488, 504, 536, 531, 520, 525, 499, 513, 532, 522, 514, 542, 490, 516, 523, 510, 574, 485, 530, 539, 517, 498, 522, 500, 486, 483, 538, 511, 525, 507, 497, 492, 503, 517, 521, 524, 520, 472, 529, 539, 483, 520, 484, 486, 488, 512, 539, 524, 521, 541, 509, 491, 490, 515, 521, 510, 471, 520, 508, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[539, 491, 504, 528, 498, 507, 501, 537, 510, 485, 535, 552, 557, 538, 521, 508, 510, 510, 499, 477, 538, 546, 540, 515, 508, 532, 549, 547, 487, 541, 497, 499, 539, 503, 513, 476, 500, 516, 494, 519, 464, 505, 500, 529, 512, 496, 540, 508, 507, 494, 490, 527, 532, 458, 500, 513, 530, 498, 513, 466, 504, 527, 476, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[497, 501, 500, 505, 535, 508, 518, 515, 526, 500, 519, 525, 497, 495, 528, 461, 532, 522, 550, 514, 525, 511, 513, 548, 506, 512, 492, 481, 519, 510, 553, 534, 509, 496, 520, 472, 512, 507, 513, 547, 516, 510, 510, 522, 536, 489, 535, 513, 539, 511, 497, 544, 498, 516, 493, 469, 557, 449, 504, 527, 527, 498, 482, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[551, 471, 506, 533, 505, 494, 519, 526, 0, 0, 0, 0, 0, 0, 0, 0, 465, 560, 533, 496, 553, 538, 499, 530, 0, 0, 0, 0, 0, 0, 0, 0, 494, 506, 491, 524, 474, 503, 533, 534, 0, 0, 0, 0, 0, 0, 0, 0, 495, 527, 499, 540, 506, 522, 536, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 514, 481, 488, 561, 494, 486, 514, 0, 0, 0, 0, 0, 0, 0, 0, 490, 510, 549, 517, 508, 510, 540, 521, 7, 6, 0, 0, 0, 0, 0, 0, 528, 475, 493, 496, 507, 503, 512, 495, 15, 8, 0, 0, 0, 0, 0, 0, 491, 479, 510, 542, 473, 516, 528, 487, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4174, 4059, 4135, 4041, 4158, 4032, 4064] avg=4096.0000 max=4174 max_rank=1 min=4032 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 516, 540, 523, 521, 502, 482, 524, 0, 0, 0, 0, 0, 0, 0, 0, 509, 532, 480, 490, 487, 509, 542, 521, 0, 0, 0, 0, 0, 0, 0, 0, 536, 506, 509, 511, 531, 494, 512, 521, 0, 0, 0, 0, 0, 0, 0, 0, 506, 491, 478, 512, 526, 491, 526, 501, 0, 0, 0, 0, 0, 0, 0, 0, 504, 508, 524, 461, 534, 567, 561, 494, 0, 0, 0, 0, 0, 0, 0, 0, 527, 507, 508, 487, 531, 495, 518, 475, 7, 6, 0, 0, 0, 0, 0, 0, 482, 518, 511, 519, 519, 521, 494, 516, 17, 9, 0, 0, 0, 0, 0, 0, 478, 506, 537, 536, 492, 501, 485, 527, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4070, 4120, 4031, 4153, 4061, 4106, 4103] avg=4096.0000 max=4153 max_rank=4 min=4031 min_rank=3 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 498, 516, 529, 509, 507, 477, 504, 0, 0, 0, 0, 0, 0, 0, 0, 473, 511, 532, 500, 473, 492, 537, 550, 0, 0, 0, 0, 0, 0, 0, 0, 541, 481, 532, 522, 502, 489, 538, 508, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 510, 526, 548, 492, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 537, 491, 543, 490, 502, 502, 502, 504, 0, 0, 0, 0, 0, 0, 0, 0, 496, 523, 506, 508, 531, 495, 503, 518, 7, 6, 0, 0, 0, 0, 0, 0, 498, 527, 519, 512, 508, 513, 502, 524, 16, 8, 0, 0, 0, 0, 0, 0, 496, 488, 482, 517, 543, 528, 516, 529, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4068, 4113, 4110, 4071, 4093, 4127, 4137] avg=4096.0000 max=4137 max_rank=7 min=4049 min_rank=0 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 546, 509, 514, 534, 542, 511, 493, 0, 0, 0, 0, 0, 0, 0, 0, 489, 524, 533, 515, 525, 500, 513, 526, 0, 0, 0, 0, 0, 0, 0, 0, 483, 524, 508, 500, 506, 516, 519, 519, 0, 0, 0, 0, 0, 0, 0, 0, 479, 533, 519, 517, 526, 518, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 524, 500, 492, 515, 480, 541, 529, 537, 0, 0, 0, 0, 0, 0, 0, 0, 502, 513, 494, 518, 487, 497, 501, 521, 7, 6, 0, 0, 0, 0, 0, 0, 509, 469, 519, 488, 506, 522, 525, 498, 17, 8, 0, 0, 0, 0, 0, 0, 506, 506, 508, 498, 505, 512, 496, 481, 40, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4155, 4125, 4075, 4136, 4118, 4046, 4061, 4052] avg=4096.0000 max=4155 max_rank=0 min=4046 min_rank=5 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16277 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[512, 505, 530, 486, 528, 523, 496, 484, 0, 0, 0, 0, 0, 0, 0, 0, 469, 521, 552, 475, 519, 526, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 485, 524, 465, 479, 464, 503, 522, 521, 0, 0, 0, 0, 0, 0, 0, 0, 499, 493, 476, 506, 544, 471, 527, 525, 0, 0, 0, 0, 0, 0, 0, 0, 531, 494, 549, 509, 523, 537, 520, 526, 0, 0, 0, 0, 0, 0, 0, 0, 534, 517, 502, 489, 545, 534, 490, 513, 7, 6, 0, 0, 0, 0, 0, 0, 530, 524, 476, 501, 496, 497, 530, 568, 15, 8, 0, 0, 0, 0, 0, 0, 535, 510, 537, 492, 511, 502, 521, 528, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4055, 3963, 4041, 4189, 4137, 4145, 4174] avg=4096.0000 max=4189 max_rank=4 min=3963 min_rank=2 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16645 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[497, 501, 500, 505, 535, 508, 518, 515, 0, 0, 0, 0, 0, 0, 0, 0, 479, 500, 519, 525, 497, 495, 528, 461, 0, 0, 0, 0, 0, 0, 0, 0, 532, 522, 550, 514, 503, 511, 513, 548, 0, 0, 0, 0, 0, 0, 0, 0, 506, 512, 492, 481, 519, 510, 553, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 496, 520, 472, 512, 507, 513, 547, 0, 0, 0, 0, 0, 0, 0, 0, 516, 510, 510, 522, 536, 489, 535, 513, 7, 6, 0, 0, 0, 0, 0, 0, 539, 511, 497, 544, 498, 516, 493, 469, 16, 8, 0, 0, 0, 0, 0, 0, 557, 449, 504, 527, 527, 498, 482, 498, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4004, 4193, 4107, 4069, 4144, 4091, 4081] avg=4096.0000 max=4193 max_rank=2 min=4004 min_rank=1 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16385 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[498, 542, 477, 510, 488, 504, 536, 531, 1, 0, 0, 0, 0, 0, 0, 0, 465, 525, 499, 513, 532, 522, 514, 542, 0, 0, 0, 0, 0, 0, 0, 0, 490, 516, 523, 510, 540, 485, 530, 539, 0, 0, 0, 0, 0, 0, 0, 0, 517, 498, 522, 500, 486, 483, 531, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 507, 497, 492, 503, 517, 521, 524, 0, 0, 0, 0, 0, 0, 0, 0, 520, 472, 529, 539, 483, 520, 484, 486, 12, 11, 6, 0, 0, 0, 0, 0, 488, 512, 539, 524, 521, 541, 509, 491, 23, 12, 0, 0, 0, 0, 0, 0, 490, 515, 521, 510, 471, 520, 508, 514, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4087, 4112, 4133, 4048, 4074, 4062, 4160, 4092] avg=4096.0000 max=4160 max_rank=6 min=4048 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[539, 491, 504, 528, 498, 507, 501, 537, 0, 0, 0, 0, 0, 0, 0, 0, 464, 485, 535, 552, 557, 538, 521, 508, 0, 0, 0, 0, 0, 0, 0, 0, 510, 510, 499, 477, 515, 546, 540, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 532, 549, 547, 487, 541, 497, 499, 0, 0, 0, 0, 0, 0, 0, 0, 532, 503, 513, 476, 500, 516, 494, 519, 0, 0, 0, 0, 0, 0, 0, 0, 464, 505, 500, 529, 512, 496, 540, 508, 7, 6, 0, 0, 0, 0, 0, 0, 507, 494, 490, 527, 532, 458, 500, 513, 17, 8, 0, 0, 0, 0, 0, 0, 530, 498, 513, 466, 504, 527, 476, 513, 38, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4160, 4112, 4160, 4053, 4067, 4046, 4065] avg=4096.0000 max=4160 max_rank=1 min=4046 min_rank=6 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16231 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3840, 4224, 4224, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 3968, 4224, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[544, 572, 497, 547, 513, 501, 490, 513, 516, 553, 529, 558, 514, 503, 490, 500, 498, 525, 502, 492, 542, 490, 556, 476, 502, 527, 527, 518, 485, 505, 492, 526, 509, 500, 513, 491, 550, 517, 522, 463, 513, 512, 509, 515, 527, 509, 509, 521, 523, 505, 511, 512, 484, 466, 515, 507, 467, 502, 504, 551, 485, 518, 520, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[491, 524, 519, 534, 522, 509, 505, 549, 521, 507, 546, 499, 519, 548, 546, 551, 548, 520, 508, 505, 491, 509, 503, 506, 476, 501, 499, 507, 505, 564, 545, 509, 501, 548, 550, 535, 533, 491, 456, 516, 482, 538, 526, 535, 533, 533, 513, 494, 522, 478, 483, 522, 487, 480, 492, 501, 500, 480, 505, 509, 493, 483, 479, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[506, 485, 520, 537, 523, 567, 510, 498, 518, 542, 531, 504, 506, 525, 476, 490, 502, 479, 542, 506, 508, 504, 567, 510, 504, 525, 510, 496, 505, 533, 522, 493, 558, 542, 493, 547, 488, 505, 483, 494, 524, 521, 497, 531, 473, 515, 482, 504, 534, 539, 490, 499, 512, 538, 492, 514, 506, 529, 517, 443, 549, 501, 485, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[507, 536, 541, 518, 491, 543, 524, 550, 507, 522, 498, 525, 516, 527, 544, 520, 514, 528, 480, 523, 506, 516, 454, 521, 518, 484, 483, 488, 519, 514, 492, 563, 510, 526, 492, 523, 496, 520, 519, 526, 495, 505, 494, 492, 543, 538, 495, 539, 514, 513, 505, 525, 481, 521, 493, 504, 512, 494, 488, 492, 491, 514, 510, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[486, 546, 505, 473, 480, 512, 526, 504, 482, 502, 509, 506, 492, 541, 504, 495, 530, 512, 501, 505, 514, 515, 544, 497, 531, 500, 510, 505, 501, 513, 518, 496, 475, 485, 511, 526, 526, 490, 508, 524, 504, 508, 509, 475, 545, 516, 550, 512, 522, 510, 499, 491, 512, 512, 510, 526, 553, 525, 507, 551, 515, 558, 522, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[495, 493, 493, 569, 500, 469, 542, 533, 517, 539, 498, 536, 516, 470, 495, 518, 520, 518, 545, 504, 486, 527, 497, 532, 524, 506, 493, 497, 498, 524, 536, 491, 464, 499, 554, 514, 504, 531, 518, 496, 496, 523, 496, 548, 488, 512, 526, 518, 512, 508, 505, 513, 502, 521, 474, 529, 531, 527, 504, 530, 519, 497, 510, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[549, 523, 499, 544, 481, 535, 490, 513, 515, 521, 494, 505, 508, 491, 541, 518, 525, 496, 505, 520, 532, 489, 516, 543, 524, 507, 542, 540, 496, 500, 485, 518, 497, 463, 484, 547, 530, 528, 473, 541, 533, 494, 527, 530, 521, 477, 486, 475, 483, 488, 540, 512, 518, 484, 530, 514, 512, 535, 510, 518, 552, 473, 525, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[550, 506, 488, 524, 553, 530, 494, 539, 516, 508, 547, 506, 506, 510, 528, 465, 504, 515, 478, 521, 554, 487, 507, 512, 546, 464, 538, 483, 523, 553, 521, 528, 500, 512, 503, 515, 532, 524, 553, 501, 537, 511, 500, 509, 528, 516, 503, 473, 514, 479, 502, 516, 494, 491, 511, 498, 512, 528, 506, 490, 490, 528, 479, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[506, 485, 520, 492, 523, 567, 510, 498, 0, 0, 0, 0, 0, 0, 0, 0, 518, 520, 531, 504, 506, 525, 476, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 479, 542, 506, 508, 504, 566, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 525, 510, 496, 505, 533, 522, 493, 1, 1, 0, 0, 0, 0, 0, 0, 558, 542, 493, 547, 488, 505, 483, 494, 2, 0, 0, 0, 0, 0, 0, 0, 524, 521, 497, 531, 473, 515, 482, 504, 0, 0, 0, 0, 0, 0, 0, 0, 534, 539, 490, 499, 512, 538, 492, 514, 45, 3, 0, 0, 0, 0, 0, 0, 506, 529, 517, 443, 549, 501, 485, 519, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4070, 4117, 4090, 4112, 4047, 4166, 4065] avg=4096.0000 max=4166 max_rank=6 min=4047 min_rank=5 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16390 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[544, 572, 497, 501, 513, 501, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 516, 530, 529, 558, 514, 503, 490, 500, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 502, 492, 542, 490, 555, 476, 0, 0, 0, 0, 0, 0, 0, 0, 502, 527, 527, 518, 485, 505, 492, 526, 1, 1, 0, 0, 0, 0, 0, 0, 509, 500, 513, 491, 550, 517, 522, 463, 3, 0, 0, 0, 0, 0, 0, 0, 513, 512, 509, 515, 527, 509, 509, 521, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 511, 512, 484, 466, 515, 507, 46, 3, 0, 0, 0, 0, 0, 0, 467, 502, 504, 551, 485, 518, 520, 515, 16, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4131, 4140, 4080, 4084, 4068, 4115, 4072, 4078] avg=4096.0000 max=4140 max_rank=1 min=4068 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16333 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[491, 524, 519, 489, 522, 509, 505, 549, 0, 0, 0, 0, 0, 0, 0, 0, 521, 488, 546, 499, 519, 548, 546, 551, 0, 0, 0, 0, 0, 0, 0, 0, 548, 520, 508, 505, 491, 509, 502, 506, 0, 0, 0, 0, 0, 0, 0, 0, 476, 501, 499, 507, 505, 564, 545, 509, 1, 1, 0, 0, 0, 0, 0, 0, 501, 548, 550, 535, 533, 491, 456, 516, 2, 0, 0, 0, 0, 0, 0, 0, 482, 538, 526, 535, 533, 533, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 522, 478, 483, 522, 487, 480, 492, 501, 45, 2, 0, 0, 0, 0, 0, 0, 500, 480, 505, 509, 493, 483, 479, 484, 14, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4218, 4089, 4108, 4132, 4154, 4012, 3947] avg=4096.0000 max=4218 max_rank=1 min=3947 min_rank=7 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16245 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[495, 493, 493, 521, 500, 469, 542, 533, 0, 0, 0, 0, 0, 0, 0, 0, 517, 518, 498, 536, 516, 470, 495, 518, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 545, 504, 486, 527, 496, 532, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 493, 497, 498, 524, 536, 491, 1, 1, 0, 0, 0, 0, 0, 0, 464, 499, 554, 514, 504, 531, 518, 496, 2, 0, 0, 0, 0, 0, 0, 0, 496, 523, 496, 548, 488, 512, 526, 518, 0, 0, 0, 0, 0, 0, 0, 0, 512, 508, 505, 513, 502, 521, 474, 529, 48, 3, 0, 0, 0, 0, 0, 0, 531, 527, 504, 530, 519, 497, 510, 518, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4068, 4128, 4071, 4082, 4107, 4115, 4151] avg=4096.0000 max=4151 max_rank=7 min=4046 min_rank=0 imbalance_ratio=1.013428 max_load_reduction=28617 max_load_reduction_pct=0.873322 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[507, 536, 541, 474, 491, 543, 524, 550, 0, 0, 0, 0, 0, 0, 0, 0, 507, 502, 498, 525, 516, 527, 544, 520, 0, 0, 0, 0, 0, 0, 0, 0, 514, 528, 480, 523, 506, 516, 454, 521, 0, 0, 0, 0, 0, 0, 0, 0, 518, 484, 483, 488, 519, 514, 492, 563, 1, 0, 0, 0, 0, 0, 0, 0, 510, 526, 492, 523, 496, 520, 519, 526, 2, 0, 0, 0, 0, 0, 0, 0, 495, 505, 494, 492, 543, 538, 495, 539, 0, 0, 0, 0, 0, 0, 0, 0, 514, 513, 505, 525, 481, 521, 493, 504, 44, 2, 0, 0, 0, 0, 0, 0, 512, 494, 488, 492, 491, 514, 510, 526, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4166, 4139, 4042, 4062, 4114, 4101, 4102, 4042] avg=4096.0000 max=4166 max_rank=0 min=4042 min_rank=2 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16359 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[486, 546, 505, 429, 480, 512, 526, 504, 0, 0, 0, 0, 0, 0, 0, 0, 482, 465, 509, 506, 492, 541, 504, 495, 0, 0, 0, 0, 0, 0, 0, 0, 530, 512, 501, 505, 514, 515, 536, 497, 0, 0, 0, 0, 0, 0, 0, 0, 531, 500, 510, 505, 501, 513, 518, 496, 4, 3, 0, 0, 0, 0, 0, 0, 475, 485, 511, 526, 526, 490, 508, 524, 8, 0, 0, 0, 0, 0, 0, 0, 504, 508, 509, 475, 545, 516, 550, 512, 5, 0, 0, 0, 0, 0, 0, 0, 522, 510, 499, 491, 512, 512, 510, 526, 44, 7, 0, 0, 0, 0, 0, 0, 553, 525, 507, 551, 515, 558, 522, 536, 18, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3988, 3994, 4110, 4081, 4053, 4124, 4133, 4285] avg=4096.0000 max=4285 max_rank=7 min=3988 min_rank=0 imbalance_ratio=1.046143 max_load_reduction=28483 max_load_reduction_pct=0.869232 moved_to_echo=16595 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[549, 523, 499, 498, 481, 535, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 501, 494, 505, 508, 491, 541, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 496, 505, 520, 532, 489, 515, 543, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 542, 540, 496, 500, 485, 518, 1, 1, 0, 0, 0, 0, 0, 0, 497, 463, 484, 547, 530, 528, 473, 541, 2, 0, 0, 0, 0, 0, 0, 0, 533, 494, 527, 530, 521, 477, 486, 475, 0, 0, 0, 0, 0, 0, 0, 0, 483, 488, 540, 512, 518, 484, 530, 514, 46, 2, 0, 0, 0, 0, 0, 0, 512, 535, 510, 518, 552, 473, 525, 503, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4073, 4125, 4114, 4065, 4043, 4117, 4143] avg=4096.0000 max=4143 max_rank=7 min=4043 min_rank=5 imbalance_ratio=1.011475 max_load_reduction=28625 max_load_reduction_pct=0.873566 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 506, 488, 480, 553, 530, 494, 539, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 547, 506, 506, 510, 528, 465, 0, 0, 0, 0, 0, 0, 0, 0, 504, 515, 478, 521, 554, 487, 506, 512, 0, 0, 0, 0, 0, 0, 0, 0, 546, 464, 538, 483, 523, 553, 521, 528, 1, 1, 0, 0, 0, 0, 0, 0, 500, 512, 503, 515, 532, 524, 553, 501, 2, 0, 0, 0, 0, 0, 0, 0, 537, 511, 500, 509, 528, 516, 503, 473, 0, 0, 0, 0, 0, 0, 0, 0, 514, 479, 502, 516, 494, 491, 511, 498, 44, 2, 0, 0, 0, 0, 0, 0, 512, 528, 506, 490, 490, 528, 479, 509, 15, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4140, 4066, 4077, 4158, 4142, 4077, 4051, 4057] avg=4096.0000 max=4158 max_rank=3 min=4051 min_rank=6 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 3968, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[481, 493, 488, 506, 518, 511, 531, 512, 542, 490, 535, 534, 526, 478, 558, 508, 540, 509, 510, 508, 506, 516, 511, 527, 485, 518, 535, 544, 505, 494, 519, 538, 537, 527, 508, 514, 524, 503, 542, 505, 523, 511, 511, 497, 515, 530, 489, 491, 494, 494, 498, 473, 486, 514, 514, 508, 486, 530, 534, 504, 489, 525, 510, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[531, 496, 538, 510, 525, 528, 514, 497, 517, 496, 510, 528, 495, 536, 532, 496, 542, 495, 491, 511, 507, 534, 550, 499, 485, 495, 500, 511, 523, 511, 507, 508, 497, 546, 561, 545, 507, 455, 503, 502, 492, 530, 518, 504, 532, 493, 535, 523, 533, 494, 495, 515, 509, 480, 505, 522, 511, 498, 503, 522, 530, 499, 493, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[509, 483, 503, 499, 506, 524, 535, 521, 529, 534, 514, 494, 495, 515, 551, 482, 511, 504, 513, 540, 502, 521, 521, 511, 536, 498, 487, 477, 509, 480, 513, 490, 516, 502, 514, 497, 511, 559, 497, 496, 498, 481, 525, 500, 498, 490, 512, 554, 499, 521, 556, 540, 543, 521, 513, 521, 534, 494, 505, 523, 507, 515, 501, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[512, 550, 532, 516, 525, 521, 510, 517, 502, 518, 494, 516, 512, 511, 523, 552, 509, 513, 504, 527, 519, 509, 493, 495, 526, 514, 508, 536, 501, 531, 554, 558, 484, 517, 519, 474, 516, 484, 458, 513, 532, 517, 483, 499, 518, 510, 518, 489, 526, 468, 551, 526, 502, 506, 486, 498, 493, 491, 558, 529, 479, 499, 526, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[481, 528, 516, 505, 539, 516, 482, 522, 527, 485, 530, 555, 498, 505, 473, 501, 529, 539, 522, 536, 475, 490, 478, 516, 515, 532, 516, 541, 523, 507, 505, 543, 507, 489, 559, 498, 515, 501, 514, 562, 509, 549, 510, 516, 519, 494, 487, 489, 498, 510, 526, 474, 551, 510, 494, 490, 521, 489, 502, 549, 511, 500, 507, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[520, 513, 561, 489, 564, 474, 469, 504, 504, 509, 490, 552, 479, 535, 522, 516, 519, 532, 482, 539, 521, 479, 503, 557, 490, 532, 511, 514, 490, 536, 470, 504, 513, 526, 514, 533, 497, 528, 536, 500, 524, 549, 493, 509, 487, 479, 503, 546, 503, 526, 545, 533, 523, 493, 485, 520, 481, 532, 535, 504, 522, 487, 466, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[571, 511, 497, 478, 525, 514, 510, 476, 534, 504, 488, 504, 548, 525, 517, 539, 489, 505, 540, 516, 494, 557, 497, 504, 482, 493, 520, 536, 508, 493, 515, 485, 500, 562, 528, 516, 510, 526, 510, 489, 513, 503, 540, 501, 526, 506, 528, 516, 487, 521, 516, 504, 508, 543, 484, 482, 524, 517, 466, 532, 482, 501, 529, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[530, 536, 479, 523, 558, 511, 509, 520, 498, 521, 538, 534, 515, 470, 512, 520, 525, 484, 511, 497, 505, 508, 538, 534, 481, 510, 463, 508, 487, 525, 499, 513, 479, 516, 492, 562, 489, 525, 523, 526, 516, 544, 493, 547, 480, 521, 535, 563, 469, 474, 519, 521, 459, 520, 465, 519, 528, 522, 513, 509, 508, 524, 501, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 493, 488, 506, 506, 511, 531, 512, 0, 0, 0, 0, 0, 0, 0, 0, 542, 490, 535, 509, 526, 478, 558, 508, 0, 0, 0, 0, 0, 0, 0, 0, 540, 509, 510, 497, 506, 516, 511, 527, 0, 0, 0, 0, 0, 0, 0, 0, 485, 518, 535, 544, 505, 494, 519, 538, 9, 5, 0, 0, 0, 0, 0, 0, 537, 527, 496, 514, 524, 503, 542, 505, 0, 0, 0, 0, 0, 0, 0, 0, 523, 506, 511, 497, 515, 530, 489, 491, 0, 0, 0, 0, 0, 0, 0, 0, 494, 494, 498, 473, 486, 514, 514, 508, 25, 9, 0, 0, 0, 0, 0, 0, 486, 530, 534, 504, 489, 525, 510, 506, 12, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4028, 4146, 4116, 4152, 4148, 4062, 4015, 4101] avg=4096.0000 max=4152 max_rank=3 min=4015 min_rank=6 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[531, 496, 538, 510, 513, 528, 514, 497, 0, 0, 0, 0, 0, 0, 0, 0, 517, 496, 510, 503, 495, 536, 532, 496, 0, 0, 0, 0, 0, 0, 0, 0, 542, 495, 491, 500, 507, 534, 550, 499, 0, 0, 0, 0, 0, 0, 0, 0, 485, 495, 500, 511, 523, 511, 507, 508, 9, 5, 0, 0, 0, 0, 0, 0, 497, 546, 548, 545, 507, 455, 503, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 525, 518, 504, 532, 493, 535, 523, 0, 0, 0, 0, 0, 0, 0, 0, 533, 494, 495, 515, 509, 480, 505, 522, 25, 10, 0, 0, 0, 0, 0, 0, 511, 498, 503, 522, 530, 499, 493, 498, 12, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4085, 4118, 4054, 4103, 4122, 4088, 4071] avg=4096.0000 max=4127 max_rank=0 min=4054 min_rank=3 imbalance_ratio=1.007568 max_load_reduction=28641 max_load_reduction_pct=0.874054 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[512, 550, 532, 516, 513, 521, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 502, 518, 494, 492, 512, 511, 523, 552, 0, 0, 0, 0, 0, 0, 0, 0, 509, 513, 504, 515, 519, 509, 493, 495, 0, 0, 0, 0, 0, 0, 0, 0, 526, 514, 508, 536, 501, 531, 554, 558, 10, 5, 0, 0, 0, 0, 0, 0, 484, 517, 506, 474, 516, 484, 458, 513, 0, 0, 0, 0, 0, 0, 0, 0, 532, 512, 483, 499, 518, 510, 518, 489, 0, 0, 0, 0, 0, 0, 0, 0, 526, 468, 551, 526, 502, 506, 486, 498, 24, 10, 0, 0, 0, 0, 0, 0, 493, 491, 558, 529, 479, 499, 526, 491, 12, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4104, 4057, 4243, 3952, 4061, 4097, 4083] avg=4096.0000 max=4243 max_rank=3 min=3952 min_rank=4 imbalance_ratio=1.035889 max_load_reduction=28525 max_load_reduction_pct=0.870514 moved_to_echo=16193 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[509, 483, 503, 499, 494, 524, 535, 521, 0, 0, 0, 0, 0, 0, 0, 0, 529, 534, 514, 471, 495, 515, 551, 482, 0, 0, 0, 0, 0, 0, 0, 0, 511, 504, 513, 528, 502, 521, 521, 511, 0, 0, 0, 0, 0, 0, 0, 0, 536, 498, 487, 477, 509, 480, 513, 490, 10, 5, 0, 0, 0, 0, 0, 0, 516, 502, 501, 497, 511, 559, 497, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 476, 525, 500, 498, 490, 512, 554, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 556, 540, 543, 521, 513, 521, 23, 10, 0, 0, 0, 0, 0, 0, 534, 494, 505, 523, 507, 515, 501, 518, 12, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4068, 4091, 4111, 4005, 4079, 4053, 4247, 4114] avg=4096.0000 max=4247 max_rank=6 min=4005 min_rank=3 imbalance_ratio=1.036865 max_load_reduction=28521 max_load_reduction_pct=0.870392 moved_to_echo=16493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[520, 513, 561, 489, 551, 474, 469, 504, 0, 0, 0, 0, 0, 0, 0, 0, 504, 509, 490, 526, 479, 535, 522, 516, 0, 0, 0, 0, 0, 0, 0, 0, 519, 532, 482, 527, 521, 479, 503, 557, 0, 0, 0, 0, 0, 0, 0, 0, 490, 532, 511, 514, 490, 536, 470, 504, 10, 5, 0, 0, 0, 0, 0, 0, 513, 526, 501, 533, 497, 528, 536, 500, 0, 0, 0, 0, 0, 0, 0, 0, 524, 544, 493, 509, 487, 479, 503, 546, 0, 0, 0, 0, 0, 0, 0, 0, 503, 526, 545, 533, 523, 493, 485, 520, 26, 10, 0, 0, 0, 0, 0, 0, 481, 532, 535, 504, 522, 487, 466, 496, 13, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4081, 4120, 4062, 4134, 4085, 4164, 4041] avg=4096.0000 max=4164 max_rank=6 min=4041 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16424 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[481, 528, 516, 505, 526, 516, 482, 522, 0, 0, 0, 0, 0, 0, 0, 0, 527, 485, 530, 529, 498, 505, 473, 501, 0, 0, 0, 0, 0, 0, 0, 0, 529, 539, 522, 524, 475, 490, 478, 516, 0, 0, 0, 0, 0, 0, 0, 0, 515, 532, 516, 541, 523, 507, 505, 543, 10, 5, 0, 0, 0, 0, 0, 0, 507, 489, 546, 498, 515, 501, 514, 562, 0, 0, 0, 0, 0, 0, 0, 0, 509, 544, 510, 516, 519, 494, 487, 489, 0, 0, 0, 0, 0, 0, 0, 0, 498, 510, 526, 474, 551, 510, 494, 490, 26, 10, 0, 0, 0, 0, 0, 0, 521, 489, 502, 549, 511, 500, 507, 488, 13, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4048, 4073, 4197, 4132, 4068, 4089, 4085] avg=4096.0000 max=4197 max_rank=3 min=4048 min_rank=1 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[571, 511, 497, 478, 507, 514, 510, 476, 0, 0, 0, 0, 0, 0, 0, 0, 534, 504, 488, 475, 548, 525, 517, 539, 0, 0, 0, 0, 0, 0, 0, 0, 489, 505, 540, 496, 494, 557, 497, 504, 0, 0, 0, 0, 0, 0, 0, 0, 482, 493, 520, 536, 508, 493, 515, 485, 13, 10, 0, 0, 0, 0, 0, 0, 500, 562, 507, 516, 510, 526, 510, 489, 0, 0, 0, 0, 0, 0, 0, 0, 513, 493, 540, 501, 526, 506, 528, 516, 0, 0, 0, 0, 0, 0, 0, 0, 487, 521, 516, 504, 508, 543, 484, 482, 29, 14, 0, 0, 0, 0, 0, 0, 524, 517, 466, 532, 482, 501, 529, 523, 18, 7, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4130, 4082, 4055, 4120, 4123, 4088, 4106] avg=4096.0000 max=4130 max_rank=1 min=4055 min_rank=3 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[530, 536, 479, 523, 545, 511, 509, 520, 0, 0, 0, 0, 0, 0, 0, 0, 498, 521, 538, 509, 515, 470, 512, 520, 0, 0, 0, 0, 0, 0, 0, 0, 525, 484, 511, 486, 505, 508, 538, 534, 0, 0, 0, 0, 0, 0, 0, 0, 481, 510, 463, 508, 487, 525, 499, 513, 9, 5, 0, 0, 0, 0, 0, 0, 479, 516, 480, 562, 489, 525, 523, 526, 0, 0, 0, 0, 0, 0, 0, 0, 516, 539, 493, 547, 480, 521, 535, 563, 0, 0, 0, 0, 0, 0, 0, 0, 469, 474, 519, 521, 459, 520, 465, 519, 25, 9, 0, 0, 0, 0, 0, 0, 528, 522, 513, 509, 508, 524, 501, 544, 13, 3, 2, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4083, 4091, 4000, 4100, 4194, 3980, 4167] avg=4096.0000 max=4194 max_rank=5 min=3980 min_rank=6 imbalance_ratio=1.023926 max_load_reduction=28574 max_load_reduction_pct=0.872009 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 3968, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[520, 525, 523, 532, 493, 503, 519, 505, 519, 488, 480, 546, 502, 508, 539, 523, 528, 554, 475, 514, 489, 522, 537, 505, 516, 518, 509, 510, 472, 536, 517, 527, 542, 522, 472, 509, 560, 505, 494, 471, 522, 525, 505, 508, 508, 504, 531, 544, 506, 484, 518, 507, 510, 488, 529, 538, 513, 467, 522, 520, 515, 489, 493, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[492, 519, 527, 499, 539, 523, 503, 503, 499, 570, 515, 480, 505, 540, 502, 513, 496, 497, 497, 505, 516, 511, 517, 503, 493, 501, 528, 488, 509, 477, 517, 491, 529, 493, 530, 525, 521, 561, 518, 479, 497, 544, 513, 497, 515, 504, 522, 508, 491, 582, 572, 462, 535, 514, 489, 512, 506, 512, 497, 507, 495, 532, 507, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[519, 541, 510, 497, 501, 519, 497, 509, 536, 517, 534, 525, 500, 536, 563, 546, 538, 503, 485, 501, 479, 522, 517, 540, 518, 518, 519, 513, 499, 508, 513, 497, 513, 501, 517, 468, 492, 505, 549, 527, 488, 512, 523, 518, 533, 500, 503, 564, 561, 459, 453, 482, 497, 534, 458, 503, 515, 508, 516, 526, 508, 520, 489, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[498, 537, 501, 507, 506, 535, 499, 518, 506, 530, 483, 502, 516, 478, 510, 510, 510, 491, 527, 494, 502, 551, 515, 546, 523, 468, 520, 512, 503, 487, 490, 548, 527, 529, 490, 536, 484, 531, 534, 494, 507, 508, 510, 491, 511, 526, 550, 505, 505, 473, 558, 516, 499, 512, 498, 507, 501, 502, 551, 502, 537, 524, 493, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[496, 519, 488, 518, 515, 516, 527, 536, 542, 503, 536, 468, 547, 500, 504, 526, 529, 495, 528, 539, 497, 552, 493, 520, 475, 523, 517, 497, 478, 507, 505, 497, 529, 493, 509, 532, 555, 518, 513, 517, 567, 488, 515, 533, 525, 497, 491, 512, 471, 479, 491, 500, 540, 517, 482, 502, 511, 523, 516, 481, 512, 520, 538, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[509, 497, 526, 536, 523, 508, 536, 492, 526, 520, 549, 572, 538, 479, 505, 486, 513, 504, 526, 495, 541, 478, 543, 467, 519, 489, 511, 489, 478, 538, 486, 470, 487, 489, 546, 523, 485, 540, 508, 501, 523, 507, 500, 543, 492, 516, 534, 497, 506, 539, 512, 510, 513, 522, 519, 511, 524, 500, 513, 500, 513, 529, 516, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[492, 484, 494, 473, 502, 534, 518, 497, 512, 544, 507, 539, 509, 484, 545, 524, 506, 505, 522, 511, 523, 542, 508, 536, 530, 550, 459, 512, 561, 493, 487, 481, 527, 509, 500, 526, 552, 519, 505, 567, 491, 499, 482, 509, 510, 483, 532, 490, 531, 483, 505, 531, 532, 543, 521, 492, 509, 503, 489, 523, 497, 492, 520, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[468, 511, 521, 522, 522, 511, 519, 560, 503, 511, 518, 473, 510, 504, 546, 487, 524, 512, 506, 550, 512, 479, 519, 495, 525, 494, 512, 488, 508, 530, 501, 485, 557, 505, 470, 556, 512, 476, 516, 547, 485, 515, 511, 516, 479, 522, 520, 495, 540, 510, 529, 462, 548, 519, 522, 517, 553, 496, 509, 525, 508, 490, 516, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 519, 527, 499, 539, 521, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 499, 570, 515, 480, 505, 540, 470, 513, 0, 0, 0, 0, 0, 0, 0, 0, 496, 497, 497, 505, 516, 501, 517, 503, 0, 0, 0, 0, 0, 0, 0, 0, 493, 501, 528, 488, 509, 477, 517, 491, 32, 19, 0, 0, 0, 0, 0, 0, 499, 493, 530, 525, 521, 561, 518, 479, 0, 0, 0, 0, 0, 0, 0, 0, 497, 544, 513, 497, 515, 504, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 491, 582, 572, 462, 535, 514, 489, 512, 11, 3, 0, 0, 0, 0, 0, 0, 506, 512, 497, 507, 495, 532, 507, 524, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4103, 4092, 4032, 4055, 4126, 4100, 4171, 4089] avg=4096.0000 max=4171 max_rank=6 min=4032 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16486 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[520, 525, 523, 532, 493, 501, 519, 505, 0, 0, 0, 0, 0, 0, 0, 0, 519, 488, 480, 546, 502, 508, 505, 523, 0, 0, 0, 0, 0, 0, 0, 0, 528, 554, 475, 514, 489, 512, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 516, 518, 509, 510, 472, 536, 517, 527, 34, 20, 0, 0, 0, 0, 0, 0, 511, 522, 472, 509, 560, 505, 494, 471, 0, 0, 0, 0, 0, 0, 0, 0, 522, 525, 505, 508, 508, 504, 531, 544, 0, 0, 0, 0, 0, 0, 0, 0, 506, 484, 518, 507, 510, 488, 529, 538, 11, 3, 0, 0, 0, 0, 0, 0, 513, 467, 522, 520, 515, 489, 493, 493, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4071, 4114, 4159, 4044, 4147, 4094, 4021] avg=4096.0000 max=4159 max_rank=3 min=4021 min_rank=7 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[509, 497, 526, 536, 523, 506, 536, 492, 0, 0, 0, 0, 0, 0, 0, 0, 526, 520, 549, 572, 538, 479, 473, 486, 0, 0, 0, 0, 0, 0, 0, 0, 513, 504, 526, 495, 541, 469, 543, 467, 0, 0, 0, 0, 0, 0, 0, 0, 519, 489, 511, 489, 478, 538, 486, 470, 32, 18, 0, 0, 0, 0, 0, 0, 459, 489, 546, 523, 485, 540, 508, 501, 0, 0, 0, 0, 0, 0, 0, 0, 523, 507, 500, 543, 492, 516, 534, 497, 0, 0, 0, 0, 0, 0, 0, 0, 506, 539, 512, 510, 513, 522, 519, 511, 10, 2, 0, 0, 0, 0, 0, 0, 524, 500, 513, 500, 513, 529, 516, 501, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4143, 4058, 4030, 4051, 4112, 4144, 4105] avg=4096.0000 max=4144 max_rank=6 min=4030 min_rank=3 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[519, 541, 510, 497, 501, 517, 497, 509, 0, 0, 0, 0, 0, 0, 0, 0, 536, 517, 534, 525, 500, 536, 527, 546, 0, 0, 0, 0, 0, 0, 0, 0, 538, 503, 485, 501, 479, 512, 517, 540, 0, 0, 0, 0, 0, 0, 0, 0, 518, 518, 519, 513, 499, 508, 513, 497, 36, 19, 0, 0, 0, 0, 0, 0, 483, 501, 517, 468, 492, 505, 549, 527, 0, 0, 0, 0, 0, 0, 0, 0, 488, 512, 523, 518, 533, 500, 503, 564, 0, 0, 0, 0, 0, 0, 0, 0, 561, 459, 453, 482, 497, 534, 458, 503, 11, 3, 0, 0, 0, 0, 0, 0, 515, 508, 516, 526, 508, 520, 489, 506, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4221, 4075, 4140, 4042, 4141, 3961, 4097] avg=4096.0000 max=4221 max_rank=1 min=3961 min_rank=6 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16241 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[496, 519, 488, 518, 515, 514, 527, 536, 0, 0, 0, 0, 0, 0, 0, 0, 542, 503, 536, 468, 547, 500, 472, 526, 0, 0, 0, 0, 0, 0, 0, 0, 529, 495, 528, 539, 497, 541, 493, 520, 0, 0, 0, 0, 0, 0, 0, 0, 475, 523, 517, 497, 478, 507, 505, 497, 32, 19, 0, 0, 0, 0, 0, 0, 499, 493, 509, 532, 555, 518, 513, 517, 0, 0, 0, 0, 0, 0, 0, 0, 567, 488, 515, 533, 525, 497, 491, 512, 0, 0, 0, 0, 0, 0, 0, 0, 471, 479, 491, 500, 540, 517, 482, 502, 11, 3, 0, 0, 0, 0, 0, 0, 511, 523, 516, 481, 512, 520, 538, 498, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4094, 4142, 4050, 4136, 4128, 3996, 4109] avg=4096.0000 max=4142 max_rank=2 min=3996 min_rank=6 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[468, 511, 521, 522, 522, 509, 519, 560, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 518, 473, 510, 504, 512, 487, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 506, 550, 512, 470, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 525, 494, 512, 488, 508, 530, 501, 485, 34, 20, 0, 0, 0, 0, 0, 0, 525, 505, 470, 556, 512, 476, 516, 547, 0, 0, 0, 0, 0, 0, 0, 0, 485, 515, 511, 516, 479, 522, 520, 495, 0, 0, 0, 0, 0, 0, 0, 0, 540, 510, 529, 462, 548, 519, 522, 517, 12, 2, 0, 0, 0, 0, 0, 0, 553, 496, 509, 525, 508, 490, 516, 516, 7, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 4018, 4088, 4097, 4107, 4043, 4161, 4122] avg=4096.0000 max=4161 max_rank=6 min=4018 min_rank=1 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16433 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[498, 537, 501, 507, 506, 533, 499, 518, 0, 0, 0, 0, 0, 0, 0, 0, 506, 530, 483, 502, 516, 478, 478, 510, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 527, 494, 502, 540, 515, 546, 0, 0, 0, 0, 0, 0, 0, 0, 523, 468, 520, 512, 503, 487, 490, 548, 32, 19, 0, 0, 0, 0, 0, 0, 497, 529, 490, 536, 484, 531, 534, 494, 0, 0, 0, 0, 0, 0, 0, 0, 507, 508, 510, 491, 511, 526, 550, 505, 0, 0, 0, 0, 0, 0, 0, 0, 505, 473, 558, 516, 499, 512, 498, 507, 11, 3, 0, 0, 0, 0, 0, 0, 501, 502, 551, 502, 537, 524, 493, 534, 8, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4099, 4003, 4125, 4102, 4095, 4108, 4082, 4154] avg=4096.0000 max=4154 max_rank=7 min=4003 min_rank=1 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16439 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[492, 484, 494, 473, 502, 527, 518, 497, 0, 0, 0, 0, 0, 0, 0, 0, 512, 544, 507, 539, 509, 484, 507, 524, 0, 0, 0, 0, 0, 0, 0, 0, 506, 505, 522, 511, 523, 523, 508, 536, 0, 0, 0, 0, 0, 0, 0, 0, 530, 550, 459, 512, 561, 493, 487, 481, 38, 24, 0, 0, 0, 0, 0, 0, 489, 509, 500, 526, 552, 519, 505, 567, 0, 0, 0, 0, 0, 0, 0, 0, 491, 499, 482, 509, 510, 483, 525, 490, 0, 0, 0, 0, 0, 0, 0, 0, 531, 483, 505, 531, 532, 543, 521, 492, 14, 7, 0, 0, 0, 0, 0, 0, 509, 503, 489, 523, 497, 492, 520, 512, 12, 7, 7, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3987, 4126, 4134, 4135, 4167, 3989, 4159, 4071] avg=4096.0000 max=4167 max_rank=4 min=3987 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[526, 488, 471, 513, 506, 537, 493, 521, 508, 515, 486, 532, 488, 528, 509, 500, 524, 525, 507, 504, 536, 485, 499, 503, 501, 514, 527, 545, 500, 510, 570, 532, 518, 486, 508, 493, 507, 511, 549, 512, 545, 560, 522, 525, 514, 460, 537, 508, 518, 505, 515, 496, 491, 478, 522, 498, 524, 498, 514, 492, 509, 509, 534, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[490, 528, 523, 542, 541, 504, 524, 497, 527, 516, 512, 484, 520, 480, 507, 506, 534, 506, 542, 491, 534, 526, 531, 547, 532, 521, 518, 524, 502, 482, 517, 500, 511, 503, 489, 524, 512, 505, 537, 532, 546, 510, 506, 462, 510, 520, 529, 499, 520, 486, 500, 497, 533, 515, 511, 489, 507, 508, 487, 502, 509, 484, 535, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[504, 532, 534, 516, 540, 527, 513, 512, 541, 502, 515, 506, 540, 516, 503, 503, 491, 508, 541, 544, 506, 530, 523, 492, 496, 477, 536, 452, 525, 511, 473, 525, 522, 506, 504, 499, 535, 519, 526, 501, 499, 490, 476, 531, 528, 503, 496, 502, 531, 551, 489, 511, 500, 519, 532, 492, 562, 501, 516, 517, 486, 505, 513, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[506, 531, 499, 517, 526, 512, 510, 553, 513, 543, 510, 514, 535, 520, 519, 551, 514, 523, 494, 513, 527, 518, 525, 486, 503, 522, 508, 505, 526, 523, 473, 479, 510, 529, 508, 502, 475, 556, 477, 518, 503, 507, 493, 480, 497, 463, 500, 512, 505, 515, 526, 551, 503, 520, 563, 471, 477, 508, 476, 485, 533, 536, 513, 558] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[512, 456, 504, 524, 540, 552, 509, 529, 507, 498, 534, 499, 513, 489, 511, 529, 538, 547, 533, 501, 513, 529, 553, 512, 507, 530, 510, 462, 516, 517, 473, 498, 486, 489, 514, 545, 484, 482, 506, 531, 518, 516, 529, 485, 506, 520, 561, 514, 507, 503, 500, 516, 525, 482, 511, 524, 516, 490, 478, 531, 547, 472, 536, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[513, 548, 486, 567, 498, 541, 527, 490, 505, 501, 519, 489, 514, 544, 533, 490, 502, 502, 507, 493, 524, 517, 498, 550, 507, 503, 549, 511, 552, 525, 504, 507, 488, 485, 527, 497, 510, 523, 509, 507, 490, 468, 525, 500, 488, 537, 513, 506, 536, 514, 543, 518, 509, 481, 542, 524, 480, 511, 524, 493, 507, 503, 499, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[521, 487, 509, 529, 528, 541, 537, 505, 506, 484, 521, 492, 519, 516, 514, 513, 515, 546, 534, 508, 528, 513, 516, 481, 501, 511, 483, 513, 504, 510, 514, 521, 539, 556, 500, 478, 506, 519, 506, 507, 524, 499, 557, 536, 523, 532, 503, 492, 486, 524, 493, 504, 508, 490, 501, 505, 535, 514, 509, 496, 493, 501, 499, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[505, 536, 512, 467, 493, 514, 512, 539, 549, 532, 549, 503, 493, 510, 509, 481, 528, 529, 544, 475, 501, 510, 502, 554, 478, 505, 527, 508, 538, 496, 502, 510, 524, 479, 536, 507, 470, 499, 481, 548, 490, 516, 504, 489, 515, 515, 522, 533, 514, 495, 514, 502, 525, 528, 494, 498, 521, 525, 508, 558, 510, 517, 501, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[504, 532, 534, 516, 540, 491, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 535, 502, 515, 506, 540, 516, 503, 503, 0, 0, 0, 0, 0, 0, 0, 0, 491, 508, 496, 544, 506, 530, 523, 492, 0, 0, 0, 0, 0, 0, 0, 0, 496, 477, 536, 452, 525, 511, 473, 525, 18, 0, 0, 0, 0, 0, 0, 0, 522, 506, 504, 499, 535, 519, 526, 501, 17, 1, 0, 0, 0, 0, 0, 0, 499, 490, 476, 531, 528, 503, 496, 502, 13, 0, 0, 0, 0, 0, 0, 0, 531, 551, 489, 511, 500, 519, 532, 492, 6, 6, 0, 0, 0, 0, 0, 0, 562, 501, 516, 517, 486, 505, 513, 472, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4120, 4090, 4013, 4130, 4038, 4137, 4098] avg=4096.0000 max=4142 max_rank=0 min=4013 min_rank=3 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[526, 488, 471, 513, 506, 501, 493, 521, 0, 0, 0, 0, 0, 0, 0, 0, 503, 515, 486, 532, 488, 528, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 524, 525, 465, 504, 536, 485, 499, 503, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 527, 545, 500, 510, 570, 532, 17, 0, 0, 0, 0, 0, 0, 0, 518, 486, 508, 493, 507, 511, 549, 512, 17, 0, 0, 0, 0, 0, 0, 0, 545, 560, 522, 525, 514, 460, 537, 508, 13, 0, 0, 0, 0, 0, 0, 0, 518, 505, 515, 496, 491, 478, 522, 498, 6, 5, 0, 0, 0, 0, 0, 0, 524, 498, 514, 492, 509, 509, 534, 507, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4061, 4041, 4216, 4101, 4184, 4034, 4112] avg=4096.0000 max=4216 max_rank=3 min=4019 min_rank=0 imbalance_ratio=1.029297 max_load_reduction=28552 max_load_reduction_pct=0.871338 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[512, 456, 504, 524, 540, 514, 509, 529, 0, 0, 0, 0, 0, 0, 0, 0, 502, 498, 534, 499, 513, 489, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 538, 547, 488, 501, 513, 529, 553, 512, 0, 0, 0, 0, 0, 0, 0, 0, 507, 530, 510, 462, 516, 517, 473, 498, 18, 0, 0, 0, 0, 0, 0, 0, 486, 489, 514, 545, 484, 482, 506, 531, 18, 1, 0, 0, 0, 0, 0, 0, 518, 516, 529, 485, 506, 520, 561, 514, 14, 0, 0, 0, 0, 0, 0, 0, 507, 503, 500, 516, 525, 482, 511, 524, 6, 5, 0, 0, 0, 0, 0, 0, 516, 490, 478, 531, 547, 472, 536, 499, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4075, 4181, 4031, 4056, 4163, 4079, 4095] avg=4096.0000 max=4181 max_rank=2 min=4031 min_rank=3 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16393 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[490, 528, 523, 542, 541, 470, 524, 497, 0, 0, 0, 0, 0, 0, 0, 0, 522, 516, 512, 484, 520, 480, 507, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 506, 497, 491, 534, 526, 531, 547, 0, 0, 0, 0, 0, 0, 0, 0, 532, 521, 518, 524, 502, 482, 517, 500, 18, 0, 0, 0, 0, 0, 0, 0, 511, 503, 489, 524, 512, 505, 537, 532, 16, 1, 0, 0, 0, 0, 0, 0, 546, 510, 506, 462, 510, 520, 529, 499, 12, 0, 0, 0, 0, 0, 0, 0, 520, 486, 500, 497, 533, 515, 511, 489, 6, 5, 0, 0, 0, 0, 0, 0, 507, 508, 487, 502, 509, 484, 535, 482, 26, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4115, 4047, 4166, 4114, 4130, 4094, 4062, 4040] avg=4096.0000 max=4166 max_rank=2 min=4040 min_rank=7 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16326 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[506, 531, 499, 517, 526, 477, 510, 553, 0, 0, 0, 0, 0, 0, 0, 0, 508, 543, 510, 514, 535, 520, 519, 551, 0, 0, 0, 0, 0, 0, 0, 0, 514, 523, 453, 513, 527, 518, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 508, 505, 526, 523, 473, 479, 17, 0, 0, 0, 0, 0, 0, 0, 510, 529, 508, 502, 475, 556, 477, 518, 16, 0, 0, 0, 0, 0, 0, 0, 503, 507, 493, 480, 497, 463, 500, 512, 13, 0, 0, 0, 0, 0, 0, 0, 505, 515, 526, 551, 503, 520, 563, 471, 6, 5, 0, 0, 0, 0, 0, 0, 477, 508, 476, 485, 533, 536, 513, 558, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4119, 4200, 4059, 4056, 4091, 3968, 4165, 4110] avg=4096.0000 max=4200 max_rank=1 min=3968 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[505, 536, 512, 467, 493, 479, 512, 539, 0, 0, 0, 0, 0, 0, 0, 0, 543, 532, 549, 503, 493, 510, 509, 481, 0, 0, 0, 0, 0, 0, 0, 0, 528, 529, 497, 475, 501, 510, 502, 554, 0, 0, 0, 0, 0, 0, 0, 0, 478, 505, 527, 508, 538, 496, 502, 510, 19, 0, 0, 0, 0, 0, 0, 0, 524, 479, 536, 507, 470, 499, 481, 548, 16, 1, 0, 0, 0, 0, 0, 0, 490, 516, 504, 489, 515, 515, 522, 533, 13, 0, 0, 0, 0, 0, 0, 0, 514, 495, 514, 502, 525, 528, 494, 498, 6, 6, 0, 0, 0, 0, 0, 0, 521, 525, 508, 558, 510, 517, 501, 519, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4120, 4096, 4083, 4061, 4097, 4082, 4186] avg=4096.0000 max=4186 max_rank=7 min=4043 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16426 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[521, 487, 509, 529, 528, 492, 537, 505, 0, 0, 0, 0, 0, 0, 0, 0, 496, 484, 521, 492, 519, 516, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 546, 477, 508, 528, 513, 516, 481, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 483, 513, 504, 510, 514, 521, 23, 0, 0, 0, 0, 0, 0, 0, 539, 556, 500, 478, 506, 519, 506, 507, 21, 4, 0, 0, 0, 0, 0, 0, 524, 499, 557, 536, 523, 532, 503, 492, 18, 0, 0, 0, 0, 0, 0, 0, 486, 524, 493, 504, 508, 490, 501, 505, 10, 10, 0, 0, 0, 0, 0, 0, 535, 514, 509, 496, 493, 501, 499, 513, 30, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4055, 4084, 4080, 4136, 4184, 4031, 4090] avg=4096.0000 max=4184 max_rank=5 min=4031 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16441 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[513, 548, 486, 567, 498, 505, 527, 490, 0, 0, 0, 0, 0, 0, 0, 0, 500, 501, 519, 489, 514, 544, 533, 490, 0, 0, 0, 0, 0, 0, 0, 0, 502, 502, 465, 493, 524, 517, 498, 550, 0, 0, 0, 0, 0, 0, 0, 0, 507, 503, 549, 511, 552, 525, 504, 507, 17, 0, 0, 0, 0, 0, 0, 0, 488, 485, 527, 497, 510, 523, 509, 507, 17, 0, 0, 0, 0, 0, 0, 0, 490, 468, 525, 500, 488, 537, 513, 506, 13, 0, 0, 0, 0, 0, 0, 0, 536, 514, 543, 518, 509, 481, 542, 524, 6, 5, 0, 0, 0, 0, 0, 0, 480, 511, 524, 493, 507, 503, 499, 495, 25, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4134, 4090, 4051, 4175, 4063, 4040, 4178, 4037] avg=4096.0000 max=4178 max_rank=6 min=4037 min_rank=7 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16318 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 3968, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4096, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[481, 481, 519, 536, 518, 504, 520, 481, 501, 520, 506, 521, 473, 551, 502, 522, 516, 504, 525, 508, 511, 495, 505, 570, 532, 501, 513, 534, 497, 497, 504, 505, 520, 505, 484, 528, 478, 504, 546, 529, 509, 487, 526, 532, 542, 455, 511, 524, 495, 511, 480, 509, 525, 550, 538, 516, 536, 534, 516, 482, 521, 488, 511, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[516, 476, 522, 518, 480, 529, 494, 507, 536, 512, 480, 494, 535, 515, 530, 510, 501, 512, 525, 500, 514, 491, 503, 525, 535, 500, 520, 507, 498, 509, 498, 519, 524, 512, 510, 517, 541, 506, 487, 491, 565, 519, 496, 556, 528, 526, 520, 500, 495, 525, 487, 524, 531, 463, 534, 534, 514, 478, 507, 486, 531, 499, 515, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[478, 478, 506, 553, 535, 515, 509, 535, 527, 486, 499, 499, 539, 524, 532, 499, 530, 472, 532, 524, 493, 543, 525, 509, 523, 517, 503, 533, 523, 532, 521, 492, 496, 536, 506, 512, 541, 458, 492, 475, 519, 495, 558, 480, 518, 511, 516, 475, 488, 507, 526, 499, 498, 547, 518, 504, 501, 491, 505, 490, 515, 557, 520, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[503, 491, 508, 532, 504, 496, 539, 540, 512, 511, 485, 496, 502, 475, 513, 501, 545, 466, 502, 529, 510, 515, 520, 509, 494, 522, 515, 491, 506, 545, 494, 538, 514, 485, 495, 514, 525, 527, 553, 513, 481, 543, 496, 545, 526, 534, 534, 527, 496, 497, 544, 511, 494, 521, 539, 480, 509, 520, 513, 495, 506, 491, 526, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[479, 481, 527, 548, 495, 519, 541, 535, 500, 533, 528, 504, 552, 468, 499, 547, 561, 520, 528, 519, 539, 526, 519, 534, 545, 529, 501, 492, 491, 482, 530, 547, 474, 499, 560, 532, 491, 479, 484, 492, 461, 502, 475, 532, 510, 517, 507, 492, 514, 477, 491, 517, 542, 549, 544, 531, 478, 505, 496, 528, 492, 504, 493, 481] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[512, 508, 494, 522, 526, 531, 542, 536, 520, 533, 512, 508, 530, 541, 532, 456, 522, 512, 517, 520, 505, 550, 509, 506, 505, 526, 499, 541, 538, 513, 535, 504, 510, 498, 488, 496, 524, 465, 494, 506, 505, 524, 512, 524, 494, 505, 507, 490, 495, 480, 484, 508, 522, 509, 525, 503, 513, 486, 512, 525, 509, 541, 480, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[488, 511, 514, 529, 536, 515, 506, 508, 525, 510, 544, 526, 512, 519, 503, 531, 526, 519, 511, 515, 495, 511, 535, 486, 529, 513, 503, 512, 503, 529, 526, 502, 531, 507, 524, 501, 490, 523, 549, 512, 495, 500, 517, 496, 519, 536, 506, 489, 490, 490, 499, 506, 513, 492, 515, 488, 508, 527, 522, 484, 515, 496, 535, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[539, 498, 532, 499, 532, 499, 489, 497, 511, 533, 537, 479, 509, 514, 543, 521, 484, 533, 519, 504, 501, 511, 498, 498, 528, 527, 494, 510, 483, 496, 488, 515, 549, 502, 565, 510, 504, 494, 517, 503, 480, 501, 520, 519, 481, 509, 538, 526, 517, 506, 524, 536, 527, 536, 531, 479, 515, 506, 483, 515, 516, 523, 510, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[481, 481, 519, 533, 518, 504, 520, 481, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 506, 521, 473, 551, 488, 522, 0, 0, 0, 0, 0, 0, 0, 0, 494, 504, 525, 508, 511, 495, 505, 570, 0, 0, 0, 0, 0, 0, 0, 0, 522, 501, 513, 534, 497, 497, 504, 505, 0, 0, 0, 0, 0, 0, 0, 0, 520, 505, 484, 528, 478, 504, 546, 529, 14, 5, 0, 0, 0, 0, 0, 0, 509, 487, 526, 532, 542, 455, 511, 524, 3, 0, 0, 0, 0, 0, 0, 0, 495, 511, 480, 509, 525, 550, 538, 516, 5, 0, 0, 0, 0, 0, 0, 0, 536, 534, 516, 482, 521, 488, 511, 523, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4037, 4082, 4112, 4073, 4113, 4089, 4129, 4133] avg=4096.0000 max=4133 max_rank=7 min=4037 min_rank=0 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16464 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[478, 478, 506, 550, 535, 515, 509, 535, 0, 0, 0, 0, 0, 0, 0, 0, 527, 486, 499, 499, 539, 524, 517, 499, 0, 0, 0, 0, 0, 0, 0, 0, 506, 472, 532, 524, 493, 543, 525, 509, 0, 0, 0, 0, 0, 0, 0, 0, 513, 517, 503, 533, 523, 532, 521, 492, 0, 0, 0, 0, 0, 0, 0, 0, 496, 536, 506, 512, 541, 458, 492, 475, 15, 5, 1, 0, 0, 0, 0, 0, 519, 495, 558, 480, 518, 511, 516, 475, 3, 0, 0, 0, 0, 0, 0, 0, 488, 507, 526, 499, 498, 547, 518, 504, 5, 0, 0, 0, 0, 0, 0, 0, 501, 491, 505, 490, 515, 557, 520, 528, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4090, 4104, 4134, 4037, 4075, 4092, 4130] avg=4096.0000 max=4134 max_rank=3 min=4037 min_rank=4 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[503, 491, 508, 529, 504, 496, 539, 540, 0, 0, 0, 0, 0, 0, 0, 0, 512, 511, 485, 496, 502, 475, 499, 501, 0, 0, 0, 0, 0, 0, 0, 0, 520, 466, 502, 529, 510, 515, 520, 509, 0, 0, 0, 0, 0, 0, 0, 0, 485, 522, 515, 491, 506, 545, 494, 538, 0, 0, 0, 0, 0, 0, 0, 0, 514, 485, 495, 514, 525, 527, 553, 513, 14, 5, 1, 0, 0, 0, 0, 0, 481, 543, 496, 545, 526, 534, 534, 527, 3, 0, 0, 0, 0, 0, 0, 0, 496, 497, 544, 511, 494, 521, 539, 480, 4, 0, 0, 0, 0, 0, 0, 0, 509, 520, 513, 495, 506, 491, 526, 505, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 3981, 4071, 4096, 4146, 4189, 4086, 4089] avg=4096.0000 max=4189 max_rank=5 min=3981 min_rank=1 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16510 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[516, 476, 522, 516, 480, 529, 494, 507, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 480, 494, 535, 515, 515, 510, 0, 0, 0, 0, 0, 0, 0, 0, 479, 512, 525, 500, 514, 491, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 525, 500, 520, 507, 498, 509, 498, 519, 0, 0, 0, 0, 0, 0, 0, 0, 524, 512, 510, 517, 541, 506, 487, 491, 15, 5, 0, 0, 0, 0, 0, 0, 565, 519, 496, 556, 528, 526, 520, 500, 2, 0, 0, 0, 0, 0, 0, 0, 495, 525, 487, 524, 531, 463, 534, 534, 5, 0, 0, 0, 0, 0, 0, 0, 514, 478, 507, 486, 531, 499, 515, 536, 22, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4097, 4049, 4076, 4108, 4212, 4098, 4088] avg=4096.0000 max=4212 max_rank=5 min=4040 min_rank=0 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[512, 508, 494, 520, 526, 531, 542, 536, 0, 0, 0, 0, 0, 0, 0, 0, 520, 533, 512, 508, 530, 541, 517, 456, 0, 0, 0, 0, 0, 0, 0, 0, 499, 512, 517, 520, 505, 550, 509, 506, 0, 0, 0, 0, 0, 0, 0, 0, 495, 526, 499, 541, 538, 513, 535, 504, 0, 0, 0, 0, 0, 0, 0, 0, 510, 498, 488, 496, 524, 465, 494, 506, 15, 5, 0, 0, 0, 0, 0, 0, 505, 524, 512, 524, 494, 505, 507, 490, 2, 0, 0, 0, 0, 0, 0, 0, 495, 480, 484, 508, 522, 509, 525, 503, 5, 0, 0, 0, 0, 0, 0, 0, 513, 486, 512, 525, 509, 541, 480, 529, 23, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4169, 4117, 4118, 4151, 4001, 4063, 4031, 4118] avg=4096.0000 max=4169 max_rank=0 min=4001 min_rank=4 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[539, 498, 532, 497, 532, 499, 489, 497, 0, 0, 0, 0, 0, 0, 0, 0, 511, 533, 537, 479, 509, 514, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 463, 533, 519, 504, 501, 511, 498, 498, 0, 0, 0, 0, 0, 0, 0, 0, 518, 527, 494, 510, 483, 496, 488, 515, 0, 0, 0, 0, 0, 0, 0, 0, 549, 502, 565, 510, 504, 494, 517, 503, 15, 5, 0, 0, 0, 0, 0, 0, 480, 501, 520, 519, 481, 509, 538, 526, 2, 0, 0, 0, 0, 0, 0, 0, 517, 506, 524, 536, 527, 536, 531, 479, 5, 0, 0, 0, 0, 0, 0, 0, 515, 506, 483, 515, 516, 523, 510, 505, 21, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4132, 4027, 4031, 4164, 4076, 4161, 4094] avg=4096.0000 max=4164 max_rank=4 min=4027 min_rank=2 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[479, 481, 527, 545, 495, 519, 541, 535, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 528, 504, 552, 468, 485, 547, 0, 0, 0, 0, 0, 0, 0, 0, 536, 520, 528, 519, 539, 526, 519, 534, 0, 0, 0, 0, 0, 0, 0, 0, 535, 529, 501, 492, 491, 482, 530, 547, 0, 0, 0, 0, 0, 0, 0, 0, 474, 499, 560, 532, 491, 479, 484, 492, 14, 5, 1, 0, 0, 0, 0, 0, 461, 502, 475, 532, 510, 517, 507, 492, 3, 0, 0, 0, 0, 0, 0, 0, 514, 477, 491, 517, 542, 549, 544, 531, 5, 0, 0, 0, 0, 0, 0, 0, 478, 505, 496, 528, 492, 504, 493, 481, 24, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4117, 4221, 4107, 4031, 3999, 4170, 4001] avg=4096.0000 max=4221 max_rank=2 min=3999 min_rank=5 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16201 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[488, 511, 514, 523, 536, 515, 506, 508, 0, 0, 0, 0, 0, 0, 0, 0, 525, 510, 544, 526, 512, 519, 485, 531, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 511, 515, 495, 511, 535, 486, 0, 0, 0, 0, 0, 0, 0, 0, 512, 513, 503, 512, 503, 529, 526, 502, 0, 0, 0, 0, 0, 0, 0, 0, 531, 507, 524, 501, 490, 523, 549, 512, 18, 8, 5, 0, 0, 0, 0, 0, 495, 500, 517, 496, 519, 536, 506, 489, 6, 1, 0, 0, 0, 0, 0, 0, 490, 490, 499, 506, 513, 492, 515, 488, 8, 0, 0, 0, 0, 0, 0, 0, 508, 527, 522, 484, 515, 496, 535, 501, 27, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4152, 4066, 4100, 4168, 4065, 4001, 4115] avg=4096.0000 max=4168 max_rank=4 min=4001 min_rank=6 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16349 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 3968, 4224, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 3968, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[531, 519, 521, 498, 513, 531, 530, 481, 498, 483, 511, 485, 471, 542, 512, 497, 490, 462, 511, 501, 537, 513, 554, 476, 551, 504, 552, 531, 538, 572, 523, 514, 512, 521, 509, 521, 504, 509, 535, 515, 503, 496, 522, 503, 535, 485, 483, 535, 566, 517, 508, 506, 480, 511, 528, 461, 475, 520, 505, 540, 518, 483, 510, 500] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[512, 517, 575, 514, 531, 526, 515, 518, 490, 523, 514, 512, 539, 532, 489, 504, 519, 541, 524, 509, 527, 487, 534, 542, 488, 500, 513, 508, 517, 508, 498, 547, 522, 487, 495, 507, 482, 506, 517, 503, 538, 501, 482, 532, 511, 518, 498, 547, 497, 521, 514, 489, 468, 522, 498, 492, 539, 516, 489, 492, 483, 524, 494, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[537, 537, 534, 498, 514, 545, 466, 518, 505, 511, 486, 500, 528, 528, 512, 509, 522, 495, 519, 511, 518, 489, 520, 524, 513, 543, 533, 521, 566, 509, 491, 549, 509, 526, 513, 517, 519, 485, 509, 526, 481, 515, 511, 556, 520, 507, 488, 468, 506, 545, 529, 463, 531, 506, 497, 512, 482, 500, 497, 486, 523, 500, 488, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[513, 501, 518, 528, 540, 511, 522, 508, 518, 543, 531, 461, 541, 526, 541, 499, 498, 491, 504, 477, 534, 490, 538, 499, 506, 482, 510, 528, 496, 551, 538, 491, 536, 544, 510, 505, 495, 462, 504, 523, 520, 503, 538, 505, 568, 529, 498, 506, 523, 524, 468, 497, 485, 475, 541, 516, 504, 486, 514, 497, 523, 529, 515, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[478, 537, 497, 482, 499, 489, 509, 504, 518, 565, 511, 539, 513, 530, 458, 481, 543, 540, 472, 509, 503, 511, 545, 532, 535, 512, 540, 496, 494, 527, 514, 495, 497, 528, 490, 504, 531, 525, 534, 534, 528, 523, 531, 526, 509, 491, 484, 520, 507, 514, 513, 528, 528, 526, 473, 498, 490, 494, 485, 516, 485, 526, 531, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[489, 487, 508, 541, 546, 507, 488, 506, 523, 464, 518, 491, 499, 503, 544, 513, 480, 507, 464, 500, 505, 502, 501, 532, 505, 550, 548, 522, 517, 492, 505, 565, 513, 533, 530, 520, 523, 516, 506, 535, 524, 480, 491, 489, 502, 501, 546, 506, 555, 485, 530, 495, 505, 491, 515, 544, 479, 521, 495, 552, 506, 518, 526, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[500, 567, 495, 519, 506, 528, 527, 523, 551, 491, 534, 537, 501, 507, 509, 500, 491, 548, 500, 476, 496, 526, 516, 508, 501, 491, 512, 572, 514, 517, 500, 484, 533, 551, 524, 504, 465, 513, 474, 519, 535, 518, 489, 498, 500, 524, 533, 481, 521, 506, 516, 506, 460, 561, 514, 508, 497, 509, 497, 513, 510, 528, 503, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[488, 512, 513, 515, 517, 514, 515, 523, 539, 521, 523, 565, 529, 495, 503, 534, 486, 520, 516, 526, 519, 503, 467, 502, 520, 515, 496, 491, 526, 522, 499, 519, 493, 494, 495, 542, 488, 496, 520, 532, 529, 523, 529, 506, 499, 492, 464, 557, 519, 472, 545, 517, 528, 521, 531, 515, 489, 511, 544, 505, 495, 479, 491, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[531, 497, 521, 498, 513, 531, 530, 481, 0, 0, 0, 0, 0, 0, 0, 0, 498, 483, 511, 485, 471, 532, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 490, 462, 511, 501, 537, 513, 554, 476, 15, 5, 0, 0, 0, 0, 0, 0, 551, 504, 498, 531, 538, 572, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 512, 521, 509, 521, 504, 509, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 522, 503, 535, 485, 483, 535, 2, 2, 0, 0, 0, 0, 0, 0, 566, 517, 508, 506, 480, 511, 528, 461, 8, 7, 0, 0, 0, 0, 0, 0, 475, 520, 505, 540, 518, 483, 510, 500, 49, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 3989, 4064, 4231, 4124, 4066, 4092, 4100] avg=4096.0000 max=4231 max_rank=3 min=3989 min_rank=1 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 495, 575, 514, 531, 526, 515, 518, 0, 0, 0, 0, 0, 0, 0, 0, 490, 523, 514, 512, 539, 522, 489, 504, 0, 0, 0, 0, 0, 0, 0, 0, 519, 541, 524, 509, 527, 487, 534, 542, 15, 5, 0, 0, 0, 0, 0, 0, 488, 500, 462, 508, 517, 508, 498, 547, 0, 0, 0, 0, 0, 0, 0, 0, 522, 487, 495, 507, 482, 506, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 538, 501, 482, 532, 511, 518, 498, 547, 2, 2, 0, 0, 0, 0, 0, 0, 497, 521, 514, 489, 468, 522, 498, 492, 8, 7, 0, 0, 0, 0, 0, 0, 539, 516, 489, 492, 483, 524, 494, 511, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4186, 4093, 4203, 4028, 4017, 4131, 4016, 4094] avg=4096.0000 max=4203 max_rank=2 min=4016 min_rank=6 imbalance_ratio=1.026123 max_load_reduction=28565 max_load_reduction_pct=0.871735 moved_to_echo=16258 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[513, 480, 518, 528, 540, 511, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 543, 531, 461, 541, 516, 541, 499, 0, 0, 0, 0, 0, 0, 0, 0, 498, 491, 504, 477, 534, 490, 538, 499, 14, 5, 0, 0, 0, 0, 0, 0, 506, 482, 460, 528, 496, 551, 538, 491, 0, 0, 0, 0, 0, 0, 0, 0, 536, 544, 510, 505, 495, 462, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 520, 503, 538, 505, 568, 529, 498, 506, 2, 2, 0, 0, 0, 0, 0, 0, 523, 524, 468, 497, 485, 475, 541, 516, 8, 7, 0, 0, 0, 0, 0, 0, 504, 486, 514, 497, 523, 529, 515, 491, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4150, 4050, 4052, 4077, 4171, 4044, 4104] avg=4096.0000 max=4171 max_rank=5 min=4044 min_rank=6 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[478, 515, 497, 482, 499, 489, 509, 504, 0, 0, 0, 0, 0, 0, 0, 0, 518, 565, 511, 539, 513, 520, 458, 481, 0, 0, 0, 0, 0, 0, 0, 0, 543, 540, 472, 509, 503, 511, 545, 532, 15, 5, 0, 0, 0, 0, 0, 0, 535, 512, 487, 496, 494, 527, 514, 495, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 490, 504, 531, 525, 534, 532, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 531, 526, 509, 491, 484, 520, 2, 2, 0, 0, 0, 0, 0, 0, 507, 514, 513, 528, 528, 526, 473, 498, 8, 7, 0, 0, 0, 0, 0, 0, 490, 494, 485, 516, 485, 526, 531, 521, 48, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3973, 4105, 4175, 4060, 4141, 4116, 4102, 4096] avg=4096.0000 max=4175 max_rank=2 min=3973 min_rank=0 imbalance_ratio=1.019287 max_load_reduction=28593 max_load_reduction_pct=0.872589 moved_to_echo=16455 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[537, 515, 534, 498, 514, 545, 466, 518, 0, 0, 0, 0, 0, 0, 0, 0, 505, 511, 486, 500, 528, 518, 512, 509, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 519, 511, 518, 489, 520, 524, 15, 5, 0, 0, 0, 0, 0, 0, 513, 543, 481, 521, 566, 509, 491, 549, 0, 0, 0, 0, 0, 0, 0, 0, 509, 526, 513, 517, 519, 485, 509, 524, 0, 0, 0, 0, 0, 0, 0, 0, 481, 515, 511, 556, 520, 507, 488, 468, 2, 2, 0, 0, 0, 0, 0, 0, 506, 545, 529, 463, 531, 506, 497, 512, 8, 7, 0, 0, 0, 0, 0, 0, 482, 500, 497, 486, 523, 500, 488, 502, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4069, 4118, 4173, 4102, 4050, 4104, 4025] avg=4096.0000 max=4173 max_rank=3 min=4025 min_rank=7 imbalance_ratio=1.018799 max_load_reduction=28595 max_load_reduction_pct=0.872650 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 490, 513, 515, 517, 514, 515, 523, 0, 0, 0, 0, 0, 0, 0, 0, 539, 521, 523, 565, 529, 487, 503, 534, 0, 0, 0, 0, 0, 0, 0, 0, 486, 520, 516, 526, 519, 503, 467, 502, 15, 4, 0, 0, 0, 0, 0, 0, 520, 515, 448, 491, 526, 522, 499, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 494, 495, 542, 488, 496, 520, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 523, 529, 506, 499, 492, 464, 557, 2, 1, 0, 0, 0, 0, 0, 0, 519, 472, 545, 517, 528, 521, 531, 515, 7, 7, 0, 0, 0, 0, 0, 0, 489, 511, 544, 505, 495, 479, 491, 514, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4201, 4058, 4040, 4058, 4102, 4162, 4072] avg=4096.0000 max=4201 max_rank=1 min=4040 min_rank=3 imbalance_ratio=1.025635 max_load_reduction=28567 max_load_reduction_pct=0.871796 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[489, 467, 508, 541, 546, 507, 488, 506, 0, 0, 0, 0, 0, 0, 0, 0, 523, 464, 518, 491, 499, 495, 544, 513, 0, 0, 0, 0, 0, 0, 0, 0, 480, 507, 464, 500, 505, 502, 501, 532, 14, 5, 0, 0, 0, 0, 0, 0, 505, 550, 494, 522, 517, 492, 505, 565, 0, 0, 0, 0, 0, 0, 0, 0, 513, 533, 530, 520, 523, 516, 506, 533, 0, 0, 0, 0, 0, 0, 0, 0, 524, 480, 491, 489, 502, 501, 546, 506, 2, 1, 0, 0, 0, 0, 0, 0, 555, 485, 530, 495, 505, 491, 515, 544, 7, 6, 0, 0, 0, 0, 0, 0, 479, 521, 495, 552, 506, 518, 526, 514, 49, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4047, 4010, 4150, 4174, 4042, 4133, 4160] avg=4096.0000 max=4174 max_rank=4 min=4010 min_rank=2 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[500, 535, 495, 519, 506, 528, 527, 523, 0, 0, 0, 0, 0, 0, 0, 0, 551, 491, 534, 537, 501, 491, 509, 500, 0, 0, 0, 0, 0, 0, 0, 0, 491, 548, 500, 476, 496, 526, 516, 508, 21, 8, 0, 0, 0, 0, 0, 0, 501, 491, 455, 572, 514, 517, 500, 484, 0, 0, 0, 0, 0, 0, 0, 0, 533, 551, 524, 504, 465, 513, 474, 511, 0, 0, 0, 0, 0, 0, 0, 0, 535, 518, 489, 498, 500, 524, 533, 481, 8, 4, 0, 0, 0, 0, 0, 0, 521, 506, 516, 506, 460, 561, 514, 508, 12, 11, 0, 0, 0, 0, 0, 0, 497, 509, 497, 513, 510, 528, 503, 511, 49, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4114, 4090, 4034, 4075, 4090, 4115, 4117] avg=4096.0000 max=4133 max_rank=0 min=4034 min_rank=3 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16397 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[473, 504, 515, 566, 482, 516, 501, 538, 527, 518, 492, 533, 520, 504, 513, 517, 525, 509, 519, 498, 482, 519, 508, 547, 499, 542, 522, 563, 510, 548, 473, 495, 472, 530, 500, 506, 487, 497, 479, 521, 515, 520, 457, 504, 501, 521, 508, 549, 513, 522, 525, 492, 486, 506, 490, 579, 505, 517, 515, 512, 484, 515, 561, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[491, 495, 496, 492, 501, 498, 535, 544, 502, 539, 520, 505, 506, 509, 530, 526, 515, 506, 487, 499, 530, 525, 496, 494, 533, 518, 480, 524, 550, 494, 532, 524, 498, 484, 559, 524, 530, 503, 535, 528, 498, 512, 505, 535, 549, 529, 545, 494, 483, 499, 481, 488, 518, 525, 526, 500, 489, 517, 490, 508, 532, 475, 493, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[542, 499, 572, 499, 533, 513, 547, 534, 529, 531, 489, 499, 515, 518, 484, 474, 495, 543, 467, 509, 524, 536, 512, 509, 512, 493, 533, 464, 538, 483, 532, 528, 535, 521, 523, 527, 510, 550, 529, 509, 513, 491, 500, 488, 515, 520, 532, 481, 487, 531, 508, 497, 509, 528, 509, 503, 503, 474, 481, 512, 503, 482, 529, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[554, 491, 490, 533, 509, 512, 522, 520, 536, 500, 504, 520, 503, 501, 511, 482, 501, 520, 529, 509, 494, 521, 509, 540, 507, 526, 479, 502, 519, 517, 480, 522, 475, 569, 536, 528, 545, 528, 484, 520, 503, 511, 532, 513, 521, 526, 517, 529, 476, 527, 505, 533, 531, 491, 495, 513, 507, 481, 498, 498, 493, 514, 523, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[495, 496, 510, 499, 523, 505, 517, 495, 499, 459, 513, 558, 489, 522, 543, 518, 514, 508, 482, 558, 524, 517, 477, 580, 515, 518, 485, 549, 523, 514, 532, 522, 541, 475, 521, 529, 497, 501, 504, 509, 537, 493, 512, 521, 524, 490, 447, 516, 537, 528, 508, 515, 477, 521, 515, 507, 490, 512, 496, 520, 507, 496, 536, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[505, 520, 523, 506, 500, 524, 519, 510, 496, 519, 519, 508, 511, 508, 547, 496, 499, 527, 506, 498, 549, 490, 534, 511, 516, 488, 524, 509, 512, 535, 539, 506, 502, 516, 527, 489, 508, 531, 527, 505, 499, 537, 517, 492, 509, 481, 513, 540, 487, 522, 510, 509, 528, 534, 498, 521, 515, 485, 533, 505, 492, 516, 480, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[511, 475, 520, 532, 530, 494, 525, 507, 519, 565, 558, 510, 491, 519, 493, 501, 518, 513, 515, 523, 532, 479, 502, 538, 469, 537, 526, 489, 514, 530, 493, 494, 511, 516, 530, 539, 456, 507, 521, 510, 534, 497, 500, 494, 514, 505, 518, 528, 509, 516, 502, 499, 511, 482, 529, 503, 526, 518, 517, 536, 511, 510, 492, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[513, 500, 497, 522, 497, 546, 547, 553, 499, 518, 554, 528, 514, 515, 544, 512, 484, 493, 509, 503, 546, 503, 546, 515, 476, 488, 506, 527, 472, 507, 480, 514, 498, 511, 525, 524, 513, 512, 497, 512, 511, 557, 452, 565, 508, 526, 493, 503, 496, 541, 512, 515, 487, 520, 516, 521, 521, 539, 494, 498, 501, 491, 490, 491] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[473, 504, 515, 566, 482, 516, 482, 538, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 492, 533, 520, 504, 498, 517, 0, 0, 0, 0, 0, 0, 0, 0, 525, 509, 519, 498, 482, 519, 508, 535, 0, 0, 0, 0, 0, 0, 0, 0, 499, 542, 522, 563, 510, 548, 473, 495, 2, 0, 0, 0, 0, 0, 0, 0, 472, 530, 484, 506, 487, 497, 479, 521, 0, 0, 0, 0, 0, 0, 0, 0, 515, 520, 457, 504, 501, 521, 508, 549, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 525, 492, 486, 506, 490, 579, 10, 3, 0, 0, 0, 0, 0, 0, 505, 517, 515, 512, 484, 515, 561, 501, 19, 16, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4076, 4109, 4095, 4154, 3976, 4075, 4126, 4157] avg=4096.0000 max=4157 max_rank=7 min=3976 min_rank=4 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16334 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[491, 495, 496, 492, 501, 498, 515, 544, 0, 0, 0, 0, 0, 0, 0, 0, 502, 539, 520, 505, 506, 509, 514, 526, 0, 0, 0, 0, 0, 0, 0, 0, 515, 506, 487, 499, 530, 525, 496, 484, 0, 0, 0, 0, 0, 0, 0, 0, 533, 518, 480, 524, 550, 494, 532, 524, 1, 0, 0, 0, 0, 0, 0, 0, 498, 484, 541, 524, 530, 503, 535, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 512, 505, 535, 549, 529, 545, 494, 0, 0, 0, 0, 0, 0, 0, 0, 483, 499, 481, 488, 518, 525, 526, 500, 9, 4, 0, 0, 0, 0, 0, 0, 489, 517, 490, 508, 532, 475, 493, 520, 20, 18, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4032, 4121, 4042, 4156, 4143, 4167, 4033, 4074] avg=4096.0000 max=4167 max_rank=5 min=4032 min_rank=0 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16417 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[554, 491, 490, 533, 509, 512, 502, 520, 0, 0, 0, 0, 0, 0, 0, 0, 536, 500, 504, 520, 503, 501, 496, 482, 0, 0, 0, 0, 0, 0, 0, 0, 501, 520, 529, 509, 494, 521, 509, 528, 0, 0, 0, 0, 0, 0, 0, 0, 507, 526, 479, 502, 519, 517, 480, 522, 2, 0, 0, 0, 0, 0, 0, 0, 475, 569, 519, 528, 545, 528, 484, 520, 0, 0, 0, 0, 0, 0, 0, 0, 503, 511, 532, 513, 521, 526, 517, 529, 0, 0, 0, 0, 0, 0, 0, 0, 476, 527, 505, 533, 531, 491, 495, 513, 10, 3, 0, 0, 0, 0, 0, 0, 507, 481, 498, 498, 493, 514, 523, 483, 20, 17, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4042, 4111, 4054, 4168, 4152, 4084, 4046] avg=4096.0000 max=4168 max_rank=4 min=4042 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16450 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[542, 499, 572, 499, 533, 513, 526, 534, 0, 0, 0, 0, 0, 0, 0, 0, 529, 531, 489, 499, 515, 518, 470, 474, 0, 0, 0, 0, 0, 0, 0, 0, 495, 543, 467, 509, 524, 536, 512, 497, 0, 0, 0, 0, 0, 0, 0, 0, 512, 493, 533, 464, 538, 483, 532, 528, 2, 0, 0, 0, 0, 0, 0, 0, 535, 521, 506, 527, 510, 550, 529, 509, 0, 0, 0, 0, 0, 0, 0, 0, 513, 491, 500, 488, 515, 520, 532, 481, 0, 0, 0, 0, 0, 0, 0, 0, 487, 531, 508, 497, 509, 528, 509, 503, 10, 3, 0, 0, 0, 0, 0, 0, 503, 474, 481, 512, 503, 482, 529, 512, 21, 17, 11, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4218, 4025, 4083, 4085, 4187, 4040, 4085, 4045] avg=4096.0000 max=4218 max_rank=0 min=4025 min_rank=1 imbalance_ratio=1.029785 max_load_reduction=28550 max_load_reduction_pct=0.871277 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[495, 496, 510, 499, 523, 505, 497, 495, 0, 0, 0, 0, 0, 0, 0, 0, 499, 459, 513, 558, 489, 522, 526, 518, 0, 0, 0, 0, 0, 0, 0, 0, 514, 508, 482, 558, 524, 517, 477, 567, 0, 0, 0, 0, 0, 0, 0, 0, 515, 518, 485, 549, 523, 514, 532, 522, 2, 0, 0, 0, 0, 0, 0, 0, 541, 475, 504, 529, 497, 501, 504, 509, 0, 0, 0, 0, 0, 0, 0, 0, 537, 493, 512, 521, 524, 490, 447, 516, 0, 0, 0, 0, 0, 0, 0, 0, 537, 528, 508, 515, 477, 521, 515, 507, 11, 4, 0, 0, 0, 0, 0, 0, 490, 512, 496, 520, 507, 496, 536, 527, 20, 17, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4020, 4084, 4147, 4160, 4060, 4040, 4123, 4134] avg=4096.0000 max=4160 max_rank=3 min=4020 min_rank=0 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 500, 497, 522, 497, 546, 526, 553, 0, 0, 0, 0, 0, 0, 0, 0, 499, 518, 554, 528, 514, 515, 527, 512, 0, 0, 0, 0, 0, 0, 0, 0, 484, 493, 509, 503, 546, 503, 546, 503, 0, 0, 0, 0, 0, 0, 0, 0, 476, 488, 506, 527, 472, 507, 480, 514, 2, 0, 0, 0, 0, 0, 0, 0, 498, 511, 508, 524, 513, 512, 497, 512, 0, 0, 0, 0, 0, 0, 0, 0, 511, 557, 452, 565, 508, 526, 493, 503, 0, 0, 0, 0, 0, 0, 0, 0, 496, 541, 512, 515, 487, 520, 516, 521, 10, 4, 0, 0, 0, 0, 0, 0, 521, 539, 494, 498, 501, 491, 490, 491, 21, 17, 13, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4167, 4087, 3972, 4075, 4115, 4122, 4076] avg=4096.0000 max=4167 max_rank=1 min=3972 min_rank=3 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16388 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[511, 475, 520, 532, 530, 494, 505, 507, 0, 0, 0, 0, 0, 0, 0, 0, 519, 565, 558, 510, 491, 519, 478, 501, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 515, 523, 532, 479, 502, 526, 0, 0, 0, 0, 0, 0, 0, 0, 469, 537, 526, 489, 514, 530, 493, 494, 2, 0, 0, 0, 0, 0, 0, 0, 511, 516, 513, 539, 456, 507, 521, 510, 0, 0, 0, 0, 0, 0, 0, 0, 534, 497, 500, 494, 514, 505, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 509, 516, 502, 499, 511, 482, 529, 503, 10, 3, 0, 0, 0, 0, 0, 0, 526, 518, 517, 536, 511, 510, 492, 505, 20, 17, 12, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4141, 4108, 4054, 4073, 4090, 4064, 4164] avg=4096.0000 max=4164 max_rank=7 min=4054 min_rank=3 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16391 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[505, 520, 523, 506, 500, 524, 495, 510, 0, 0, 0, 0, 0, 0, 0, 0, 496, 519, 519, 508, 511, 508, 522, 496, 0, 0, 0, 0, 0, 0, 0, 0, 499, 527, 506, 498, 549, 490, 534, 492, 0, 0, 0, 0, 0, 0, 0, 0, 516, 488, 524, 509, 512, 535, 539, 506, 4, 0, 0, 0, 0, 0, 0, 0, 502, 516, 508, 489, 508, 531, 527, 505, 0, 0, 0, 0, 0, 0, 0, 0, 499, 537, 517, 492, 509, 481, 513, 540, 1, 0, 0, 0, 0, 0, 0, 0, 487, 522, 510, 509, 528, 534, 498, 521, 14, 8, 0, 0, 0, 0, 0, 0, 515, 485, 533, 505, 492, 516, 480, 486, 24, 19, 17, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4079, 4095, 4133, 4086, 4089, 4131, 4072] avg=4096.0000 max=4133 max_rank=3 min=4072 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True + [2026-08-26 14:57:43] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 283.1 | throughput per GPU (TFLOP/s/GPU): 436.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.535187E+00 | loss scale: 1.0 | grad norm: 3.657 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[503, 556, 508, 534, 478, 477, 523, 546, 522, 518, 518, 526, 515, 499, 524, 507, 501, 464, 499, 509, 531, 524, 485, 498, 509, 500, 535, 503, 521, 532, 494, 493, 510, 483, 517, 489, 538, 542, 529, 530, 504, 506, 529, 505, 546, 490, 489, 545, 496, 504, 514, 489, 549, 488, 507, 509, 503, 546, 516, 520, 531, 517, 481, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[494, 515, 531, 503, 513, 515, 529, 542, 518, 501, 502, 505, 540, 531, 501, 515, 503, 495, 540, 457, 441, 507, 514, 534, 533, 486, 508, 481, 538, 489, 511, 509, 512, 518, 485, 519, 536, 500, 554, 545, 536, 516, 541, 478, 503, 478, 505, 538, 526, 560, 475, 487, 539, 513, 479, 505, 511, 526, 484, 534, 529, 512, 495, 528] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[453, 488, 560, 511, 520, 518, 525, 533, 512, 522, 510, 547, 533, 475, 521, 506, 489, 519, 500, 502, 516, 506, 495, 520, 521, 534, 492, 528, 519, 506, 523, 472, 493, 525, 535, 501, 512, 535, 512, 499, 519, 482, 525, 514, 573, 527, 468, 553, 516, 492, 497, 478, 541, 496, 503, 492, 523, 492, 493, 511, 521, 523, 529, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[502, 495, 505, 534, 563, 516, 509, 530, 541, 515, 513, 499, 482, 532, 484, 543, 536, 519, 538, 495, 529, 520, 531, 541, 486, 526, 501, 499, 523, 481, 513, 497, 514, 542, 504, 502, 523, 490, 472, 504, 484, 532, 530, 518, 503, 504, 486, 501, 546, 516, 520, 528, 535, 536, 486, 469, 501, 503, 497, 492, 523, 504, 503, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[491, 513, 503, 515, 518, 482, 500, 527, 494, 497, 522, 537, 515, 513, 529, 531, 589, 517, 517, 513, 546, 524, 528, 538, 503, 502, 520, 505, 526, 504, 509, 493, 541, 538, 519, 514, 520, 527, 505, 479, 512, 483, 491, 519, 502, 510, 472, 484, 527, 510, 517, 496, 470, 524, 508, 489, 503, 529, 527, 515, 500, 481, 546, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[538, 524, 516, 520, 532, 503, 509, 542, 523, 475, 481, 521, 481, 506, 519, 523, 476, 479, 510, 531, 517, 508, 517, 506, 485, 533, 500, 551, 508, 499, 519, 539, 508, 506, 527, 524, 481, 506, 488, 488, 549, 525, 483, 529, 532, 513, 558, 519, 482, 463, 566, 518, 489, 511, 502, 529, 517, 477, 465, 505, 519, 527, 532, 539] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[502, 520, 542, 528, 518, 518, 538, 513, 521, 520, 525, 517, 525, 519, 524, 536, 546, 506, 517, 499, 502, 513, 523, 523, 460, 532, 513, 505, 488, 505, 523, 520, 525, 515, 541, 473, 491, 520, 509, 536, 494, 568, 515, 450, 521, 501, 536, 520, 480, 502, 543, 482, 506, 504, 499, 507, 496, 504, 474, 510, 503, 518, 508, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[526, 514, 538, 513, 486, 504, 494, 518, 516, 493, 533, 530, 504, 509, 474, 527, 521, 515, 526, 512, 513, 538, 493, 502, 554, 521, 521, 528, 501, 482, 543, 505, 472, 519, 496, 549, 496, 522, 485, 500, 481, 529, 517, 478, 534, 499, 510, 499, 508, 492, 555, 515, 544, 515, 512, 470, 497, 511, 540, 544, 492, 493, 529, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 495, 505, 534, 563, 516, 509, 498, 0, 0, 0, 0, 0, 0, 0, 0, 541, 515, 513, 499, 482, 532, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 536, 519, 538, 495, 529, 520, 531, 534, 0, 0, 0, 0, 0, 0, 0, 0, 486, 526, 501, 499, 523, 481, 513, 497, 7, 4, 1, 0, 0, 0, 0, 0, 514, 541, 504, 502, 523, 490, 472, 504, 0, 0, 0, 0, 0, 0, 0, 0, 484, 532, 530, 518, 503, 504, 486, 501, 0, 0, 0, 0, 0, 0, 0, 0, 546, 516, 520, 528, 535, 536, 486, 469, 30, 0, 0, 0, 0, 0, 0, 0, 501, 503, 497, 492, 523, 504, 503, 502, 14, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4091, 4202, 4038, 4050, 4058, 4166, 4041] avg=4096.0000 max=4202 max_rank=2 min=4038 min_rank=3 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16315 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[503, 556, 508, 534, 478, 477, 523, 512, 0, 0, 0, 0, 0, 0, 0, 0, 522, 518, 518, 526, 515, 499, 524, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 464, 499, 509, 531, 524, 485, 492, 0, 0, 0, 0, 0, 0, 0, 0, 509, 500, 535, 503, 521, 532, 494, 493, 6, 4, 1, 0, 0, 0, 0, 0, 510, 482, 517, 489, 538, 542, 529, 530, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 529, 505, 546, 490, 489, 545, 0, 0, 0, 0, 0, 0, 0, 0, 496, 504, 514, 489, 549, 488, 507, 509, 31, 0, 0, 0, 0, 0, 0, 0, 503, 546, 516, 520, 531, 517, 481, 494, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4112, 4005, 4098, 4137, 4114, 4087, 4124] avg=4096.0000 max=4137 max_rank=4 min=4005 min_rank=2 imbalance_ratio=1.010010 max_load_reduction=28631 max_load_reduction_pct=0.873749 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[453, 488, 560, 511, 520, 518, 525, 500, 0, 0, 0, 0, 0, 0, 0, 0, 512, 522, 510, 547, 533, 475, 521, 489, 0, 0, 0, 0, 0, 0, 0, 0, 489, 519, 500, 502, 516, 506, 495, 514, 0, 0, 0, 0, 0, 0, 0, 0, 521, 534, 492, 528, 519, 506, 523, 472, 6, 4, 1, 0, 0, 0, 0, 0, 493, 524, 535, 501, 512, 535, 512, 499, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 525, 514, 573, 527, 468, 553, 0, 0, 0, 0, 0, 0, 0, 0, 516, 492, 497, 478, 541, 496, 503, 492, 30, 0, 0, 0, 0, 0, 0, 0, 523, 492, 493, 511, 521, 523, 529, 512, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4109, 4041, 4106, 4111, 4161, 4045, 4120] avg=4096.0000 max=4161 max_rank=5 min=4041 min_rank=2 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[526, 514, 538, 513, 486, 504, 494, 487, 0, 0, 0, 0, 0, 0, 0, 0, 516, 493, 533, 530, 504, 509, 474, 510, 0, 0, 0, 0, 0, 0, 0, 0, 521, 515, 526, 512, 513, 538, 493, 496, 0, 0, 0, 0, 0, 0, 0, 0, 554, 521, 521, 528, 501, 482, 543, 505, 6, 4, 1, 0, 0, 0, 0, 0, 472, 518, 496, 549, 496, 522, 485, 500, 0, 0, 0, 0, 0, 0, 0, 0, 481, 529, 517, 478, 534, 499, 510, 499, 0, 0, 0, 0, 0, 0, 0, 0, 508, 492, 555, 515, 544, 515, 512, 470, 29, 0, 0, 0, 0, 0, 0, 0, 497, 511, 540, 544, 492, 493, 529, 511, 13, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4062, 4069, 4114, 4166, 4038, 4047, 4140, 4132] avg=4096.0000 max=4166 max_rank=3 min=4038 min_rank=4 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16357 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[502, 520, 542, 528, 518, 518, 538, 482, 0, 0, 0, 0, 0, 0, 0, 0, 521, 520, 525, 517, 525, 519, 524, 518, 0, 0, 0, 0, 0, 0, 0, 0, 546, 506, 517, 499, 502, 513, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 460, 532, 513, 505, 488, 505, 523, 520, 6, 4, 1, 0, 0, 0, 0, 0, 525, 514, 541, 473, 491, 520, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 494, 568, 515, 450, 521, 501, 536, 520, 0, 0, 0, 0, 0, 0, 0, 0, 480, 502, 543, 482, 506, 504, 499, 507, 29, 0, 0, 0, 0, 0, 0, 0, 496, 504, 474, 510, 503, 518, 508, 476, 14, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4169, 4123, 4057, 4109, 4105, 4052, 4005] avg=4096.0000 max=4169 max_rank=1 min=4005 min_rank=7 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16271 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[538, 524, 516, 520, 532, 503, 509, 509, 0, 0, 0, 0, 0, 0, 0, 0, 523, 475, 481, 521, 481, 506, 519, 506, 0, 0, 0, 0, 0, 0, 0, 0, 476, 479, 510, 531, 517, 508, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 485, 533, 500, 551, 508, 499, 519, 539, 6, 4, 1, 0, 0, 0, 0, 0, 508, 505, 527, 524, 481, 506, 488, 488, 0, 0, 0, 0, 0, 0, 0, 0, 549, 525, 483, 529, 532, 513, 558, 519, 0, 0, 0, 0, 0, 0, 0, 0, 482, 463, 566, 518, 489, 511, 502, 529, 30, 0, 0, 0, 0, 0, 0, 0, 517, 477, 465, 505, 519, 527, 532, 539, 13, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4151, 4012, 4038, 4145, 4027, 4208, 4090, 4097] avg=4096.0000 max=4208 max_rank=5 min=4012 min_rank=1 imbalance_ratio=1.027344 max_load_reduction=28560 max_load_reduction_pct=0.871582 moved_to_echo=16422 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[494, 515, 531, 503, 513, 515, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 518, 501, 502, 505, 540, 531, 501, 488, 0, 0, 0, 0, 0, 0, 0, 0, 503, 495, 540, 457, 441, 507, 514, 523, 0, 0, 0, 0, 0, 0, 0, 0, 533, 486, 508, 481, 538, 489, 511, 509, 11, 10, 8, 0, 0, 0, 0, 0, 512, 503, 485, 519, 536, 500, 554, 545, 0, 0, 0, 0, 0, 0, 0, 0, 536, 516, 541, 478, 503, 478, 505, 538, 7, 0, 0, 0, 0, 0, 0, 0, 526, 560, 475, 487, 539, 513, 479, 505, 33, 0, 0, 0, 0, 0, 0, 0, 511, 526, 484, 534, 529, 512, 495, 528, 17, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4086, 3980, 4084, 4154, 4102, 4117, 4143] avg=4096.0000 max=4154 max_rank=4 min=3980 min_rank=2 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16516 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[491, 513, 503, 515, 518, 482, 500, 495, 0, 0, 0, 0, 0, 0, 0, 0, 494, 497, 522, 537, 515, 513, 529, 513, 0, 0, 0, 0, 0, 0, 0, 0, 589, 517, 517, 513, 546, 524, 528, 531, 0, 0, 0, 0, 0, 0, 0, 0, 503, 502, 520, 505, 526, 504, 509, 493, 7, 4, 1, 0, 0, 0, 0, 0, 541, 537, 519, 514, 520, 527, 505, 479, 0, 0, 0, 0, 0, 0, 0, 0, 512, 483, 491, 519, 502, 510, 472, 484, 0, 0, 0, 0, 0, 0, 0, 0, 527, 510, 517, 496, 470, 524, 508, 489, 30, 0, 0, 0, 0, 0, 0, 0, 503, 529, 527, 515, 500, 481, 546, 489, 14, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4017, 4120, 4265, 4074, 4142, 3973, 4071, 4106] avg=4096.0000 max=4265 max_rank=2 min=3973 min_rank=5 imbalance_ratio=1.041260 max_load_reduction=28503 max_load_reduction_pct=0.869843 moved_to_echo=16292 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[483, 503, 486, 548, 527, 515, 509, 467, 560, 530, 503, 575, 547, 548, 517, 516, 542, 537, 517, 473, 501, 555, 529, 514, 535, 509, 487, 542, 528, 478, 475, 476, 505, 498, 490, 487, 451, 490, 505, 485, 528, 494, 499, 556, 541, 507, 489, 538, 536, 559, 494, 500, 484, 536, 512, 507, 507, 496, 520, 496, 515, 503, 520, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[494, 538, 480, 547, 526, 526, 546, 513, 516, 520, 469, 517, 498, 488, 490, 517, 493, 523, 516, 506, 515, 557, 547, 521, 517, 520, 535, 523, 502, 516, 530, 526, 534, 527, 495, 473, 542, 542, 518, 504, 480, 496, 470, 507, 502, 495, 516, 513, 488, 515, 529, 541, 524, 456, 532, 525, 491, 447, 541, 496, 537, 530, 495, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[514, 507, 495, 538, 536, 515, 492, 527, 500, 510, 546, 510, 524, 541, 536, 515, 508, 503, 510, 509, 505, 498, 498, 486, 510, 494, 533, 533, 520, 524, 541, 514, 513, 519, 536, 519, 491, 522, 547, 490, 504, 521, 494, 510, 509, 509, 464, 519, 484, 496, 527, 510, 521, 481, 511, 496, 508, 516, 507, 497, 498, 494, 548, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[555, 495, 491, 541, 500, 510, 552, 517, 524, 507, 499, 500, 489, 494, 500, 516, 524, 516, 496, 546, 520, 501, 559, 519, 510, 541, 516, 498, 511, 487, 486, 474, 527, 536, 533, 514, 536, 522, 518, 521, 480, 511, 497, 491, 530, 486, 450, 486, 496, 487, 563, 493, 515, 519, 503, 505, 537, 489, 542, 508, 544, 528, 484, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[515, 507, 506, 532, 516, 524, 515, 491, 562, 485, 484, 524, 485, 529, 531, 489, 492, 480, 480, 523, 526, 492, 512, 498, 509, 527, 481, 523, 503, 523, 504, 532, 503, 497, 504, 538, 491, 541, 516, 519, 519, 507, 530, 485, 500, 524, 547, 529, 510, 533, 474, 511, 498, 536, 527, 504, 492, 551, 500, 494, 584, 515, 465, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[506, 491, 492, 507, 502, 517, 496, 494, 525, 528, 537, 539, 511, 461, 509, 517, 477, 475, 502, 528, 530, 512, 503, 506, 524, 528, 502, 515, 549, 484, 498, 509, 510, 454, 527, 530, 547, 511, 514, 499, 529, 518, 503, 537, 488, 507, 502, 499, 526, 505, 503, 523, 556, 500, 538, 542, 516, 495, 518, 499, 518, 526, 523, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[485, 516, 489, 518, 494, 548, 497, 544, 541, 494, 502, 516, 543, 502, 538, 491, 520, 524, 498, 561, 487, 527, 464, 491, 534, 515, 491, 481, 522, 537, 483, 508, 503, 517, 522, 518, 513, 496, 517, 521, 498, 532, 521, 551, 525, 506, 547, 480, 481, 519, 526, 546, 516, 484, 476, 509, 539, 520, 493, 458, 515, 535, 544, 479] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[486, 508, 482, 531, 476, 519, 521, 531, 520, 535, 508, 540, 529, 494, 506, 540, 500, 499, 536, 498, 512, 518, 581, 548, 488, 499, 514, 514, 503, 495, 491, 506, 500, 495, 523, 519, 537, 517, 538, 515, 540, 489, 533, 487, 486, 491, 522, 516, 500, 524, 493, 503, 501, 504, 508, 508, 491, 519, 523, 524, 517, 491, 530, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[514, 507, 495, 538, 536, 515, 492, 527, 2, 0, 0, 0, 0, 0, 0, 0, 470, 510, 546, 510, 524, 541, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 503, 510, 509, 505, 498, 490, 486, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 533, 533, 520, 524, 541, 514, 5, 4, 0, 0, 0, 0, 0, 0, 513, 519, 536, 519, 491, 522, 538, 490, 0, 0, 0, 0, 0, 0, 0, 0, 504, 521, 494, 510, 509, 509, 464, 519, 26, 0, 0, 0, 0, 0, 0, 0, 484, 496, 527, 510, 521, 481, 511, 496, 5, 0, 0, 0, 0, 0, 0, 0, 508, 516, 507, 497, 498, 494, 548, 515, 3, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4126, 4152, 4009, 4178, 4128, 4056, 4031, 4088] avg=4096.0000 max=4178 max_rank=3 min=4009 min_rank=2 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16303 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[494, 538, 480, 547, 526, 526, 546, 513, 2, 0, 0, 0, 0, 0, 0, 0, 485, 520, 469, 517, 498, 488, 490, 517, 0, 0, 0, 0, 0, 0, 0, 0, 493, 523, 516, 506, 515, 557, 538, 521, 0, 0, 0, 0, 0, 0, 0, 0, 517, 520, 535, 523, 502, 516, 530, 526, 5, 4, 0, 0, 0, 0, 0, 0, 534, 527, 495, 473, 542, 542, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 480, 496, 470, 507, 502, 495, 516, 513, 27, 0, 0, 0, 0, 0, 0, 0, 488, 515, 529, 541, 524, 456, 532, 525, 4, 0, 0, 0, 0, 0, 0, 0, 491, 447, 541, 496, 537, 530, 495, 475, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 3984, 4169, 4178, 4128, 4006, 4114, 4017] avg=4096.0000 max=4178 max_rank=3 min=3984 min_rank=1 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[483, 503, 486, 548, 527, 515, 509, 467, 2, 0, 0, 0, 0, 0, 0, 0, 525, 530, 503, 575, 547, 548, 517, 516, 0, 0, 0, 0, 0, 0, 0, 0, 542, 537, 517, 473, 501, 555, 520, 514, 0, 0, 0, 0, 0, 0, 0, 0, 535, 509, 487, 542, 528, 478, 475, 476, 5, 5, 0, 0, 0, 0, 0, 0, 505, 498, 490, 487, 451, 490, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 499, 556, 541, 507, 489, 538, 30, 0, 0, 0, 0, 0, 0, 0, 536, 559, 494, 500, 484, 536, 512, 507, 4, 0, 0, 0, 0, 0, 0, 0, 507, 496, 520, 496, 515, 503, 520, 488, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4040, 4261, 4159, 4040, 3904, 4182, 4132, 4050] avg=4096.0000 max=4261 max_rank=1 min=3904 min_rank=4 imbalance_ratio=1.040283 max_load_reduction=28507 max_load_reduction_pct=0.869965 moved_to_echo=16268 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[515, 507, 506, 532, 516, 524, 515, 491, 2, 0, 0, 0, 0, 0, 0, 0, 527, 485, 484, 524, 485, 529, 531, 489, 0, 0, 0, 0, 0, 0, 0, 0, 492, 480, 480, 523, 526, 492, 504, 498, 0, 0, 0, 0, 0, 0, 0, 0, 509, 527, 481, 523, 503, 523, 504, 532, 5, 5, 0, 0, 0, 0, 0, 0, 503, 497, 504, 538, 491, 541, 509, 519, 0, 0, 0, 0, 0, 0, 0, 0, 519, 507, 530, 485, 500, 524, 547, 529, 30, 0, 0, 0, 0, 0, 0, 0, 510, 533, 474, 511, 498, 536, 527, 504, 4, 0, 0, 0, 0, 0, 0, 0, 492, 551, 500, 494, 584, 515, 465, 524, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4054, 3995, 4112, 4102, 4171, 4097, 4129] avg=4096.0000 max=4171 max_rank=5 min=3995 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16499 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[555, 495, 491, 541, 500, 510, 552, 517, 2, 0, 0, 0, 0, 0, 0, 0, 491, 507, 499, 500, 489, 494, 500, 516, 0, 0, 0, 0, 0, 0, 0, 0, 524, 516, 496, 546, 520, 501, 550, 519, 0, 0, 0, 0, 0, 0, 0, 0, 510, 541, 516, 498, 511, 487, 486, 474, 5, 5, 0, 0, 0, 0, 0, 0, 527, 536, 533, 514, 536, 522, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 480, 511, 497, 491, 530, 486, 450, 486, 28, 0, 0, 0, 0, 0, 0, 0, 496, 487, 563, 493, 515, 519, 503, 505, 4, 0, 0, 0, 0, 0, 0, 0, 537, 489, 542, 508, 544, 528, 484, 523, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4163, 3996, 4172, 4033, 4200, 3959, 4085, 4160] avg=4096.0000 max=4200 max_rank=4 min=3959 min_rank=5 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[485, 516, 489, 518, 494, 548, 497, 544, 2, 0, 0, 0, 0, 0, 0, 0, 507, 494, 502, 516, 543, 502, 538, 491, 0, 0, 0, 0, 0, 0, 0, 0, 520, 524, 498, 561, 487, 527, 457, 491, 0, 0, 0, 0, 0, 0, 0, 0, 534, 515, 491, 481, 522, 537, 483, 508, 4, 5, 0, 0, 0, 0, 0, 0, 503, 517, 522, 518, 513, 496, 510, 521, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 521, 551, 525, 506, 547, 480, 29, 0, 0, 0, 0, 0, 0, 0, 481, 519, 526, 546, 516, 484, 476, 509, 4, 0, 0, 0, 0, 0, 0, 0, 539, 520, 493, 458, 515, 535, 544, 479, 3, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4093, 4093, 4065, 4080, 4100, 4189, 4061, 4087] avg=4096.0000 max=4189 max_rank=5 min=4061 min_rank=6 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16437 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[506, 491, 492, 507, 502, 517, 496, 494, 5, 0, 0, 0, 0, 0, 0, 0, 486, 528, 537, 539, 511, 461, 509, 517, 0, 0, 0, 0, 0, 0, 0, 0, 477, 475, 502, 528, 530, 512, 487, 506, 0, 0, 0, 0, 0, 0, 0, 0, 524, 528, 502, 515, 549, 484, 498, 509, 9, 8, 0, 0, 0, 0, 0, 0, 510, 454, 527, 530, 547, 511, 493, 499, 0, 0, 0, 0, 0, 0, 0, 0, 529, 518, 503, 537, 488, 507, 502, 499, 31, 0, 0, 0, 0, 0, 0, 0, 526, 505, 503, 523, 556, 500, 538, 542, 9, 0, 0, 0, 0, 0, 0, 0, 516, 495, 518, 499, 518, 526, 523, 531, 7, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4010, 4088, 4017, 4126, 4071, 4114, 4202, 4140] avg=4096.0000 max=4202 max_rank=6 min=4010 min_rank=0 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16527 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[486, 508, 482, 531, 476, 519, 521, 531, 2, 0, 0, 0, 0, 0, 0, 0, 488, 535, 508, 540, 529, 494, 506, 540, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 536, 498, 512, 518, 571, 548, 0, 0, 0, 0, 0, 0, 0, 0, 488, 499, 514, 514, 503, 495, 491, 506, 6, 5, 0, 0, 0, 0, 0, 0, 500, 495, 523, 519, 537, 517, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 540, 489, 533, 487, 486, 491, 522, 516, 27, 0, 0, 0, 0, 0, 0, 0, 500, 524, 493, 503, 501, 504, 508, 508, 5, 0, 0, 0, 0, 0, 0, 0, 491, 519, 523, 524, 517, 491, 530, 496, 4, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4140, 4182, 4021, 4135, 4091, 4046, 4097] avg=4096.0000 max=4182 max_rank=2 min=4021 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4096, 4352, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 3968, 4352, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[478, 533, 491, 481, 520, 536, 529, 499, 483, 548, 510, 500, 548, 514, 489, 473, 515, 539, 517, 508, 508, 500, 538, 513, 476, 511, 519, 526, 500, 501, 492, 506, 517, 509, 509, 530, 515, 505, 484, 484, 475, 497, 474, 516, 538, 544, 522, 543, 547, 533, 493, 522, 573, 495, 500, 497, 504, 507, 518, 542, 512, 498, 567, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[480, 510, 550, 480, 526, 502, 473, 533, 559, 506, 522, 481, 523, 477, 565, 513, 527, 527, 502, 531, 509, 506, 514, 498, 490, 518, 510, 505, 511, 530, 501, 523, 523, 528, 491, 537, 545, 513, 532, 472, 527, 489, 514, 496, 507, 512, 505, 518, 500, 526, 516, 496, 487, 474, 522, 529, 531, 520, 488, 490, 546, 534, 504, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[496, 505, 492, 495, 533, 509, 524, 542, 523, 475, 533, 492, 481, 517, 480, 498, 533, 526, 498, 502, 505, 490, 502, 506, 546, 505, 535, 496, 532, 480, 535, 517, 516, 550, 504, 492, 522, 495, 515, 500, 544, 539, 506, 543, 505, 506, 535, 492, 497, 483, 508, 498, 503, 505, 545, 520, 512, 492, 528, 521, 514, 544, 527, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[537, 495, 502, 471, 498, 505, 511, 514, 562, 501, 527, 515, 552, 538, 525, 479, 496, 535, 501, 525, 500, 513, 520, 531, 492, 501, 533, 513, 518, 551, 494, 532, 511, 502, 521, 509, 492, 528, 494, 507, 503, 521, 546, 506, 531, 503, 506, 534, 527, 501, 505, 486, 525, 504, 500, 478, 510, 494, 460, 510, 514, 537, 510, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[508, 537, 491, 502, 493, 476, 502, 517, 528, 541, 490, 488, 506, 534, 524, 516, 492, 490, 508, 521, 486, 498, 515, 489, 528, 546, 479, 529, 559, 525, 514, 513, 526, 505, 511, 525, 511, 520, 542, 508, 484, 499, 502, 519, 510, 491, 494, 521, 529, 497, 544, 524, 492, 508, 508, 523, 529, 489, 509, 507, 527, 516, 518, 535] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[496, 555, 537, 520, 533, 510, 459, 502, 504, 510, 520, 521, 489, 534, 540, 514, 510, 491, 540, 514, 536, 489, 504, 517, 526, 500, 496, 496, 481, 504, 514, 490, 493, 498, 528, 529, 498, 484, 558, 498, 552, 519, 511, 462, 511, 494, 515, 544, 527, 539, 556, 542, 515, 523, 490, 485, 481, 495, 514, 521, 530, 507, 511, 486] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[515, 520, 543, 509, 517, 495, 498, 521, 509, 503, 515, 496, 502, 520, 518, 522, 491, 509, 519, 469, 502, 517, 469, 512, 531, 539, 485, 511, 517, 513, 500, 544, 544, 512, 505, 519, 496, 542, 516, 490, 477, 541, 519, 485, 493, 523, 498, 535, 501, 499, 522, 499, 511, 528, 553, 532, 518, 527, 531, 502, 507, 519, 485, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[509, 515, 495, 518, 522, 488, 483, 510, 531, 513, 548, 529, 488, 484, 521, 504, 519, 495, 504, 485, 521, 535, 530, 522, 518, 511, 486, 520, 506, 538, 537, 498, 535, 527, 505, 461, 499, 539, 547, 544, 511, 495, 529, 519, 494, 504, 530, 543, 531, 479, 486, 511, 550, 472, 491, 504, 508, 519, 482, 511, 493, 508, 520, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[480, 510, 550, 480, 526, 502, 473, 533, 13, 12, 2, 0, 0, 0, 0, 0, 546, 506, 522, 481, 523, 477, 565, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 502, 531, 509, 506, 514, 498, 8, 6, 2, 0, 0, 0, 0, 0, 490, 518, 510, 505, 511, 520, 501, 523, 0, 0, 0, 0, 0, 0, 0, 0, 523, 528, 491, 537, 545, 513, 520, 472, 0, 0, 0, 0, 0, 0, 0, 0, 527, 489, 514, 496, 507, 512, 505, 512, 0, 0, 0, 0, 0, 0, 0, 0, 500, 526, 516, 496, 487, 474, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 531, 520, 488, 490, 546, 532, 504, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4133, 4130, 4078, 4129, 4062, 4050, 4105] avg=4096.0000 max=4133 max_rank=1 min=4050 min_rank=6 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[478, 533, 491, 481, 520, 536, 529, 499, 11, 11, 2, 0, 0, 0, 0, 0, 472, 548, 510, 500, 548, 514, 489, 473, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 517, 508, 508, 500, 538, 513, 7, 6, 1, 0, 0, 0, 0, 0, 476, 511, 519, 526, 500, 492, 492, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 509, 509, 530, 515, 505, 473, 484, 0, 0, 0, 0, 0, 0, 0, 0, 475, 497, 474, 516, 538, 544, 522, 537, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 493, 522, 573, 495, 500, 497, 0, 0, 0, 0, 0, 0, 0, 0, 504, 507, 518, 542, 512, 497, 567, 497, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4054, 4152, 4022, 4042, 4103, 4160, 4144] avg=4096.0000 max=4160 max_rank=6 min=4022 min_rank=3 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[508, 537, 491, 502, 493, 476, 502, 517, 12, 12, 2, 0, 0, 0, 0, 0, 516, 541, 490, 488, 506, 534, 524, 516, 0, 0, 0, 0, 0, 0, 0, 0, 492, 490, 508, 521, 486, 498, 515, 489, 8, 6, 1, 0, 0, 0, 0, 0, 528, 546, 479, 529, 559, 515, 514, 513, 0, 0, 0, 0, 0, 0, 0, 0, 526, 505, 511, 525, 511, 520, 530, 508, 0, 0, 0, 0, 0, 0, 0, 0, 484, 499, 502, 519, 510, 491, 494, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 497, 544, 524, 492, 508, 508, 523, 0, 0, 0, 0, 0, 0, 0, 0, 529, 489, 509, 507, 527, 515, 518, 535, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4052, 4115, 4014, 4183, 4136, 4014, 4125, 4129] avg=4096.0000 max=4183 max_rank=3 min=4014 min_rank=2 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[496, 505, 492, 495, 533, 509, 524, 542, 12, 12, 2, 0, 0, 0, 0, 0, 511, 475, 533, 492, 481, 517, 480, 498, 0, 0, 0, 0, 0, 0, 0, 0, 533, 526, 498, 502, 505, 490, 502, 506, 7, 6, 2, 0, 0, 0, 0, 0, 546, 505, 535, 496, 532, 471, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 516, 550, 504, 492, 522, 495, 503, 500, 0, 0, 0, 0, 0, 0, 0, 0, 544, 539, 506, 543, 505, 506, 535, 486, 0, 0, 0, 0, 0, 0, 0, 0, 497, 483, 508, 498, 503, 505, 545, 520, 0, 0, 0, 0, 0, 0, 0, 0, 512, 492, 528, 521, 514, 542, 527, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 3987, 4077, 4137, 4082, 4164, 4059, 4140] avg=4096.0000 max=4164 max_rank=5 min=3987 min_rank=1 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16445 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[496, 555, 537, 520, 533, 510, 459, 502, 12, 13, 2, 0, 0, 0, 0, 0, 492, 510, 520, 521, 489, 534, 540, 514, 0, 0, 0, 0, 0, 0, 0, 0, 510, 491, 540, 514, 536, 489, 504, 517, 7, 6, 1, 0, 0, 0, 0, 0, 526, 500, 496, 496, 481, 495, 514, 490, 0, 0, 0, 0, 0, 0, 0, 0, 493, 498, 528, 529, 498, 484, 545, 498, 0, 0, 0, 0, 0, 0, 0, 0, 552, 519, 511, 462, 511, 494, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 527, 539, 556, 542, 515, 523, 490, 485, 0, 0, 0, 0, 0, 0, 0, 0, 481, 495, 514, 521, 530, 506, 511, 486, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4120, 4115, 3998, 4073, 4102, 4177, 4044] avg=4096.0000 max=4177 max_rank=6 min=3998 min_rank=3 imbalance_ratio=1.019775 max_load_reduction=28591 max_load_reduction_pct=0.872528 moved_to_echo=16396 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[537, 495, 502, 471, 498, 505, 511, 514, 13, 11, 2, 0, 0, 0, 0, 0, 549, 501, 527, 515, 552, 538, 525, 479, 0, 0, 0, 0, 0, 0, 0, 0, 496, 535, 501, 525, 500, 513, 520, 531, 8, 6, 2, 0, 0, 0, 0, 0, 492, 501, 533, 513, 518, 541, 494, 532, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 521, 509, 492, 528, 483, 507, 0, 0, 0, 0, 0, 0, 0, 0, 503, 521, 546, 506, 531, 503, 506, 528, 0, 0, 0, 0, 0, 0, 0, 0, 527, 501, 505, 486, 525, 504, 500, 478, 0, 0, 0, 0, 0, 0, 0, 0, 510, 494, 460, 510, 514, 535, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4186, 4137, 4124, 4053, 4144, 4026, 4039] avg=4096.0000 max=4186 max_rank=1 min=4026 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16262 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[515, 520, 543, 509, 517, 495, 498, 521, 17, 16, 6, 0, 0, 0, 0, 0, 492, 503, 515, 496, 502, 520, 518, 522, 0, 0, 0, 0, 0, 0, 0, 0, 491, 509, 519, 469, 502, 517, 469, 512, 12, 11, 6, 0, 0, 0, 0, 0, 531, 539, 485, 511, 517, 495, 500, 544, 0, 0, 0, 0, 0, 0, 0, 0, 544, 512, 505, 519, 496, 542, 500, 490, 0, 0, 0, 0, 0, 0, 0, 0, 477, 541, 519, 485, 493, 523, 498, 524, 0, 0, 0, 0, 0, 0, 0, 0, 501, 499, 522, 499, 511, 528, 553, 532, 2, 0, 0, 0, 0, 0, 0, 0, 518, 527, 531, 502, 507, 511, 485, 508, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4068, 4017, 4122, 4108, 4060, 4147, 4089] avg=4096.0000 max=4157 max_rank=0 min=4017 min_rank=2 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 515, 495, 518, 522, 488, 483, 510, 13, 12, 2, 0, 0, 0, 0, 0, 518, 513, 548, 529, 488, 484, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 519, 495, 504, 485, 521, 535, 530, 522, 8, 6, 1, 0, 0, 0, 0, 0, 518, 511, 486, 520, 506, 528, 537, 498, 0, 0, 0, 0, 0, 0, 0, 0, 535, 527, 505, 461, 499, 539, 535, 544, 0, 0, 0, 0, 0, 0, 0, 0, 511, 495, 529, 519, 494, 504, 530, 537, 0, 0, 0, 0, 0, 0, 0, 0, 531, 479, 486, 511, 550, 472, 491, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 519, 482, 511, 493, 507, 520, 538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4105, 4126, 4104, 4145, 4119, 4024, 4078] avg=4096.0000 max=4145 max_rank=4 min=4024 min_rank=6 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[511, 504, 529, 497, 546, 502, 487, 496, 525, 530, 505, 484, 491, 494, 524, 500, 517, 520, 524, 510, 522, 501, 536, 505, 536, 510, 499, 523, 524, 498, 550, 515, 511, 486, 503, 493, 515, 491, 532, 506, 512, 554, 502, 531, 516, 511, 519, 485, 536, 497, 506, 486, 553, 496, 501, 493, 558, 476, 515, 520, 494, 522, 514, 519] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[504, 547, 516, 508, 525, 474, 519, 503, 500, 531, 534, 503, 527, 521, 546, 508, 532, 547, 527, 486, 532, 525, 524, 513, 492, 484, 543, 517, 537, 487, 489, 521, 523, 496, 502, 476, 505, 489, 534, 456, 495, 535, 494, 502, 515, 519, 513, 526, 496, 522, 517, 539, 507, 470, 524, 496, 518, 509, 503, 542, 487, 550, 472, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[509, 521, 538, 535, 517, 534, 527, 472, 507, 499, 506, 539, 522, 514, 512, 518, 527, 546, 492, 521, 537, 510, 543, 514, 472, 528, 491, 493, 498, 538, 512, 506, 538, 526, 519, 456, 503, 519, 493, 515, 513, 495, 506, 511, 508, 475, 495, 518, 529, 526, 527, 487, 512, 511, 511, 503, 501, 508, 492, 489, 527, 501, 514, 542] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[517, 535, 500, 501, 543, 492, 510, 547, 502, 533, 534, 521, 509, 525, 510, 517, 453, 526, 517, 539, 525, 510, 485, 540, 541, 515, 487, 485, 522, 508, 506, 513, 505, 517, 506, 521, 537, 529, 539, 495, 485, 543, 503, 518, 496, 551, 490, 551, 508, 474, 510, 495, 510, 530, 486, 504, 504, 493, 516, 497, 482, 490, 533, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[525, 519, 526, 507, 503, 552, 550, 508, 513, 511, 510, 529, 519, 515, 517, 535, 487, 501, 527, 493, 535, 500, 547, 512, 525, 553, 527, 516, 530, 492, 452, 517, 500, 495, 498, 530, 532, 498, 545, 506, 543, 528, 483, 446, 519, 493, 510, 502, 521, 471, 521, 481, 496, 529, 533, 537, 481, 498, 544, 490, 487, 515, 488, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[578, 541, 484, 465, 511, 508, 491, 528, 518, 552, 501, 476, 489, 516, 537, 501, 533, 502, 481, 540, 488, 480, 521, 517, 527, 527, 510, 512, 490, 544, 509, 491, 530, 493, 527, 508, 559, 488, 505, 502, 507, 527, 502, 504, 501, 536, 520, 527, 531, 515, 513, 537, 507, 523, 507, 518, 494, 477, 514, 485, 514, 512, 528, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[499, 535, 484, 509, 506, 524, 527, 489, 510, 528, 512, 492, 501, 536, 524, 495, 528, 536, 541, 483, 547, 476, 547, 537, 504, 503, 542, 517, 519, 507, 510, 498, 507, 535, 504, 493, 512, 504, 509, 529, 534, 545, 483, 482, 526, 524, 466, 495, 537, 477, 540, 542, 546, 507, 485, 510, 503, 517, 527, 550, 475, 476, 465, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[512, 536, 532, 514, 553, 531, 500, 499, 500, 540, 481, 543, 483, 515, 522, 500, 535, 543, 504, 538, 487, 531, 500, 514, 503, 517, 501, 510, 463, 548, 526, 492, 542, 529, 520, 492, 538, 507, 480, 469, 484, 510, 504, 496, 481, 562, 532, 501, 470, 507, 514, 494, 490, 529, 520, 498, 518, 509, 514, 529, 545, 508, 499, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[511, 477, 529, 497, 546, 502, 487, 496, 0, 0, 0, 0, 0, 0, 0, 0, 525, 513, 505, 484, 491, 494, 524, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 478, 524, 510, 522, 501, 536, 505, 0, 0, 0, 0, 0, 0, 0, 0, 536, 510, 499, 523, 524, 498, 550, 515, 5, 0, 0, 0, 0, 0, 0, 0, 511, 486, 503, 493, 515, 491, 532, 506, 15, 1, 0, 0, 0, 0, 0, 0, 512, 554, 502, 531, 516, 511, 519, 485, 12, 0, 0, 0, 0, 0, 0, 0, 536, 497, 506, 486, 553, 496, 501, 493, 12, 0, 0, 0, 0, 0, 0, 0, 558, 476, 515, 520, 494, 522, 514, 519, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4036, 4093, 4160, 4053, 4142, 4080, 4159] avg=4096.0000 max=4160 max_rank=3 min=4036 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16434 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[504, 517, 516, 508, 525, 474, 519, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 514, 534, 503, 527, 521, 546, 508, 0, 0, 0, 0, 0, 0, 0, 0, 532, 503, 527, 486, 532, 525, 524, 513, 0, 0, 0, 0, 0, 0, 0, 0, 492, 484, 543, 517, 537, 487, 489, 521, 5, 0, 0, 0, 0, 0, 0, 0, 523, 496, 502, 476, 505, 489, 534, 456, 17, 1, 0, 0, 0, 0, 0, 0, 495, 535, 494, 502, 515, 519, 513, 526, 13, 0, 0, 0, 0, 0, 0, 0, 496, 522, 517, 539, 507, 470, 524, 496, 12, 0, 0, 0, 0, 0, 0, 0, 518, 509, 503, 542, 487, 550, 472, 514, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4153, 4142, 4075, 3999, 4112, 4083, 4138] avg=4096.0000 max=4153 max_rank=1 min=3999 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[517, 506, 500, 501, 543, 492, 510, 547, 0, 0, 0, 0, 0, 0, 0, 0, 502, 516, 534, 521, 509, 525, 510, 517, 0, 0, 0, 0, 0, 0, 0, 0, 453, 484, 517, 539, 525, 510, 485, 540, 0, 0, 0, 0, 0, 0, 0, 0, 541, 515, 487, 485, 522, 508, 506, 513, 5, 0, 0, 0, 0, 0, 0, 0, 505, 517, 506, 521, 537, 529, 539, 495, 16, 1, 0, 0, 0, 0, 0, 0, 485, 543, 503, 518, 496, 551, 490, 551, 13, 0, 0, 0, 0, 0, 0, 0, 508, 474, 510, 495, 510, 530, 486, 504, 12, 0, 0, 0, 0, 0, 0, 0, 504, 493, 516, 497, 482, 490, 533, 482, 41, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4116, 4134, 4053, 4082, 4166, 4150, 4029, 4038] avg=4096.0000 max=4166 max_rank=4 min=4029 min_rank=6 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[509, 492, 538, 535, 517, 534, 527, 472, 0, 0, 0, 0, 0, 0, 0, 0, 507, 483, 506, 539, 522, 514, 512, 518, 0, 0, 0, 0, 0, 0, 0, 0, 527, 502, 492, 521, 537, 510, 543, 514, 0, 0, 0, 0, 0, 0, 0, 0, 472, 528, 491, 493, 498, 538, 512, 506, 5, 0, 0, 0, 0, 0, 0, 0, 538, 526, 519, 456, 503, 519, 493, 515, 16, 1, 0, 0, 0, 0, 0, 0, 513, 495, 506, 511, 508, 475, 495, 518, 13, 0, 0, 0, 0, 0, 0, 0, 529, 526, 527, 487, 512, 511, 511, 503, 11, 0, 0, 0, 0, 0, 0, 0, 501, 508, 492, 489, 527, 501, 514, 542, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4124, 4101, 4146, 4043, 4086, 4034, 4117, 4117] avg=4096.0000 max=4146 max_rank=2 min=4034 min_rank=5 imbalance_ratio=1.012207 max_load_reduction=28622 max_load_reduction_pct=0.873474 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[525, 490, 526, 507, 503, 552, 550, 508, 0, 0, 0, 0, 0, 0, 0, 0, 513, 495, 510, 529, 519, 515, 517, 535, 0, 0, 0, 0, 0, 0, 0, 0, 487, 461, 527, 493, 535, 500, 547, 512, 0, 0, 0, 0, 0, 0, 0, 0, 525, 553, 527, 516, 530, 492, 452, 517, 5, 0, 0, 0, 0, 0, 0, 0, 500, 495, 498, 530, 532, 498, 545, 506, 16, 1, 0, 0, 0, 0, 0, 0, 543, 528, 483, 446, 519, 493, 510, 502, 13, 0, 0, 0, 0, 0, 0, 0, 521, 471, 521, 481, 496, 529, 533, 537, 11, 0, 0, 0, 0, 0, 0, 0, 481, 498, 544, 490, 487, 515, 488, 495, 39, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4133, 4062, 4117, 4121, 4037, 4100, 4037] avg=4096.0000 max=4161 max_rank=0 min=4037 min_rank=5 imbalance_ratio=1.015869 max_load_reduction=28607 max_load_reduction_pct=0.873016 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[499, 506, 484, 509, 506, 524, 527, 489, 0, 0, 0, 0, 0, 0, 0, 0, 510, 511, 512, 492, 501, 536, 524, 495, 0, 0, 0, 0, 0, 0, 0, 0, 528, 493, 541, 483, 547, 476, 547, 537, 0, 0, 0, 0, 0, 0, 0, 0, 504, 503, 542, 517, 519, 507, 510, 498, 5, 0, 0, 0, 0, 0, 0, 0, 507, 535, 504, 493, 512, 504, 509, 529, 16, 1, 0, 0, 0, 0, 0, 0, 534, 545, 483, 482, 526, 524, 466, 495, 13, 0, 0, 0, 0, 0, 0, 0, 537, 477, 540, 542, 546, 507, 485, 510, 12, 0, 0, 0, 0, 0, 0, 0, 503, 517, 527, 550, 475, 476, 465, 497, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4081, 4152, 4105, 4110, 4068, 4156, 4052] avg=4096.0000 max=4156 max_rank=6 min=4044 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[578, 499, 484, 465, 511, 508, 491, 528, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 501, 476, 489, 516, 537, 501, 0, 0, 0, 0, 0, 0, 0, 0, 533, 452, 481, 540, 488, 480, 521, 517, 0, 0, 0, 0, 0, 0, 0, 0, 527, 527, 510, 512, 490, 544, 509, 491, 11, 0, 0, 0, 0, 0, 0, 0, 530, 493, 527, 508, 559, 488, 505, 502, 22, 5, 0, 0, 0, 0, 0, 0, 507, 527, 502, 504, 501, 536, 520, 527, 18, 0, 0, 0, 0, 0, 0, 0, 531, 515, 513, 537, 507, 523, 507, 518, 16, 2, 0, 0, 0, 0, 0, 0, 494, 477, 514, 485, 514, 512, 528, 489, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4063, 4012, 4121, 4139, 4142, 4169, 4058] avg=4096.0000 max=4169 max_rank=6 min=4012 min_rank=2 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16508 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[512, 507, 532, 514, 553, 531, 500, 499, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 481, 543, 483, 515, 522, 500, 0, 0, 0, 0, 0, 0, 0, 0, 535, 499, 504, 538, 487, 531, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 503, 517, 501, 510, 463, 548, 526, 492, 5, 0, 0, 0, 0, 0, 0, 0, 542, 529, 520, 492, 538, 507, 480, 469, 16, 1, 0, 0, 0, 0, 0, 0, 484, 510, 504, 496, 481, 562, 532, 501, 13, 0, 0, 0, 0, 0, 0, 0, 470, 507, 514, 494, 490, 529, 520, 498, 12, 0, 0, 0, 0, 0, 0, 0, 518, 509, 514, 529, 545, 508, 499, 504, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4148, 4067, 4108, 4065, 4094, 4083, 4034, 4169] avg=4096.0000 max=4169 max_rank=7 min=4034 min_rank=6 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16380 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4352, 4096, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 3968, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[529, 502, 529, 492, 484, 521, 514, 575, 516, 516, 524, 482, 534, 524, 509, 488, 507, 531, 527, 531, 524, 495, 512, 514, 514, 489, 511, 517, 509, 507, 518, 467, 536, 527, 514, 502, 518, 496, 516, 490, 527, 501, 497, 519, 518, 528, 512, 498, 505, 501, 488, 499, 560, 506, 504, 526, 519, 505, 534, 499, 469, 490, 557, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[490, 493, 533, 531, 550, 496, 474, 498, 506, 529, 491, 523, 489, 529, 505, 488, 475, 496, 490, 508, 502, 520, 522, 493, 532, 513, 542, 511, 511, 500, 491, 513, 528, 500, 506, 583, 506, 542, 530, 517, 543, 501, 491, 491, 536, 507, 537, 515, 509, 532, 547, 512, 530, 528, 493, 482, 518, 513, 505, 480, 471, 509, 510, 552] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[516, 530, 489, 507, 485, 504, 517, 501, 531, 512, 489, 515, 544, 511, 513, 512, 501, 536, 496, 500, 528, 505, 543, 511, 531, 517, 522, 530, 533, 500, 517, 519, 493, 493, 517, 517, 470, 507, 553, 500, 500, 487, 478, 506, 510, 491, 523, 534, 528, 502, 525, 531, 511, 515, 488, 528, 513, 515, 492, 499, 543, 506, 504, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[491, 492, 509, 503, 498, 521, 483, 516, 567, 484, 529, 498, 511, 468, 553, 566, 473, 537, 509, 509, 539, 515, 496, 533, 522, 483, 526, 505, 494, 514, 528, 529, 508, 513, 524, 497, 496, 529, 513, 531, 505, 498, 538, 548, 502, 523, 495, 517, 494, 499, 495, 480, 476, 494, 507, 536, 503, 515, 526, 508, 500, 545, 512, 540] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[500, 515, 516, 539, 490, 520, 520, 501, 481, 504, 476, 525, 512, 538, 506, 524, 512, 491, 483, 501, 485, 536, 522, 505, 512, 533, 490, 512, 525, 519, 536, 531, 520, 514, 499, 511, 495, 523, 468, 516, 522, 554, 482, 524, 520, 506, 490, 506, 505, 512, 490, 488, 571, 532, 549, 498, 518, 551, 514, 549, 482, 505, 491, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[523, 527, 537, 486, 500, 494, 516, 523, 486, 522, 536, 499, 510, 518, 483, 531, 535, 489, 508, 496, 495, 548, 506, 505, 506, 517, 532, 484, 515, 517, 529, 533, 525, 478, 545, 524, 529, 468, 484, 493, 472, 503, 492, 498, 511, 500, 508, 512, 531, 515, 518, 537, 492, 536, 541, 472, 512, 538, 517, 526, 510, 515, 542, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[491, 529, 511, 507, 524, 510, 558, 495, 537, 552, 512, 496, 524, 515, 508, 463, 542, 486, 532, 520, 495, 491, 484, 500, 561, 520, 511, 498, 509, 516, 510, 506, 508, 520, 522, 547, 473, 479, 524, 494, 505, 506, 479, 522, 532, 503, 556, 522, 512, 525, 537, 470, 524, 528, 522, 494, 508, 491, 513, 532, 497, 518, 517, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[541, 548, 499, 524, 465, 544, 553, 484, 516, 502, 495, 561, 532, 513, 550, 494, 521, 530, 504, 473, 524, 526, 496, 511, 543, 477, 512, 507, 526, 483, 543, 502, 528, 516, 505, 488, 571, 505, 504, 509, 486, 502, 488, 526, 509, 513, 510, 501, 524, 549, 464, 481, 522, 501, 551, 494, 500, 499, 527, 489, 505, 505, 510, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[529, 502, 529, 492, 484, 521, 514, 575, 0, 0, 0, 0, 0, 0, 0, 0, 516, 516, 524, 482, 521, 524, 509, 488, 0, 0, 0, 0, 0, 0, 0, 0, 507, 531, 527, 531, 524, 495, 512, 514, 16, 0, 0, 0, 0, 0, 0, 0, 496, 489, 511, 517, 509, 507, 518, 467, 0, 0, 0, 0, 0, 0, 0, 0, 536, 527, 514, 502, 518, 496, 516, 490, 1, 0, 0, 0, 0, 0, 0, 0, 527, 501, 497, 519, 518, 528, 512, 498, 13, 2, 0, 0, 0, 0, 0, 0, 505, 501, 488, 499, 555, 506, 504, 526, 0, 0, 0, 0, 0, 0, 0, 0, 519, 505, 534, 499, 469, 490, 557, 495, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4080, 4157, 4014, 4100, 4115, 4084, 4072] avg=4096.0000 max=4157 max_rank=2 min=4014 min_rank=3 imbalance_ratio=1.014893 max_load_reduction=28611 max_load_reduction_pct=0.873138 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 492, 509, 503, 498, 521, 483, 516, 0, 0, 0, 0, 0, 0, 0, 0, 567, 484, 529, 498, 498, 468, 553, 566, 0, 0, 0, 0, 0, 0, 0, 0, 473, 537, 509, 509, 539, 515, 496, 533, 17, 0, 0, 0, 0, 0, 0, 0, 503, 483, 526, 505, 494, 514, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0, 508, 513, 524, 497, 496, 529, 513, 531, 1, 0, 0, 0, 0, 0, 0, 0, 505, 498, 538, 548, 502, 523, 495, 517, 13, 2, 0, 0, 0, 0, 0, 0, 494, 499, 495, 480, 472, 494, 507, 536, 0, 0, 0, 0, 0, 0, 0, 0, 503, 515, 526, 508, 500, 545, 512, 540, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4013, 4163, 4128, 4082, 4112, 4141, 3977, 4152] avg=4096.0000 max=4163 max_rank=1 min=3977 min_rank=6 imbalance_ratio=1.016357 max_load_reduction=28605 max_load_reduction_pct=0.872955 moved_to_echo=16382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 515, 516, 539, 490, 520, 520, 501, 0, 0, 0, 0, 0, 0, 0, 0, 481, 504, 476, 525, 499, 538, 506, 524, 0, 0, 0, 0, 0, 0, 0, 0, 512, 491, 483, 501, 485, 536, 522, 505, 16, 0, 0, 0, 0, 0, 0, 0, 494, 533, 490, 512, 525, 519, 536, 531, 0, 0, 0, 0, 0, 0, 0, 0, 520, 514, 499, 511, 495, 523, 468, 516, 1, 0, 0, 0, 0, 0, 0, 0, 522, 554, 482, 524, 520, 506, 490, 506, 13, 2, 0, 0, 0, 0, 0, 0, 505, 512, 490, 488, 566, 532, 549, 498, 0, 0, 0, 0, 0, 0, 0, 0, 518, 551, 514, 549, 482, 505, 491, 503, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4101, 4053, 4051, 4140, 4047, 4119, 4140, 4117] avg=4096.0000 max=4140 max_rank=3 min=4047 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[516, 530, 489, 507, 485, 504, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 531, 512, 489, 515, 530, 511, 513, 512, 0, 0, 0, 0, 0, 0, 0, 0, 501, 536, 496, 500, 528, 505, 543, 511, 17, 0, 0, 0, 0, 0, 0, 0, 511, 517, 522, 530, 533, 500, 517, 519, 0, 0, 0, 0, 0, 0, 0, 0, 493, 493, 517, 517, 470, 507, 553, 500, 1, 0, 0, 0, 0, 0, 0, 0, 500, 487, 478, 506, 510, 491, 523, 534, 14, 3, 0, 0, 0, 0, 0, 0, 528, 502, 525, 531, 507, 515, 488, 528, 0, 0, 0, 0, 0, 0, 0, 0, 513, 515, 492, 499, 543, 506, 504, 524, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4049, 4113, 4137, 4149, 4051, 4046, 4124, 4099] avg=4096.0000 max=4149 max_rank=3 min=4046 min_rank=5 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16320 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[490, 493, 533, 531, 550, 496, 474, 498, 0, 0, 0, 0, 0, 0, 0, 0, 506, 529, 491, 523, 477, 529, 505, 488, 0, 0, 0, 0, 0, 0, 0, 0, 475, 496, 490, 508, 502, 520, 522, 493, 17, 0, 0, 0, 0, 0, 0, 0, 512, 513, 542, 511, 511, 500, 491, 513, 0, 0, 0, 0, 0, 0, 0, 0, 528, 500, 506, 583, 506, 542, 530, 517, 1, 0, 0, 0, 0, 0, 0, 0, 543, 501, 491, 491, 536, 507, 537, 515, 12, 3, 0, 0, 0, 0, 0, 0, 509, 532, 547, 512, 525, 528, 493, 482, 0, 0, 0, 0, 0, 0, 0, 0, 518, 513, 505, 480, 471, 509, 510, 552, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4048, 4023, 4093, 4213, 4136, 4128, 4062] avg=4096.0000 max=4213 max_rank=4 min=4023 min_rank=2 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16539 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[541, 548, 499, 524, 465, 544, 553, 484, 0, 0, 0, 0, 0, 0, 0, 0, 516, 502, 495, 561, 519, 513, 550, 494, 0, 0, 0, 0, 0, 0, 0, 0, 521, 530, 504, 473, 524, 526, 496, 511, 17, 0, 0, 0, 0, 0, 0, 0, 523, 477, 512, 507, 526, 483, 543, 502, 0, 0, 0, 0, 0, 0, 0, 0, 528, 516, 505, 488, 571, 505, 504, 509, 1, 0, 0, 0, 0, 0, 0, 0, 486, 502, 488, 526, 509, 513, 510, 501, 13, 3, 0, 0, 0, 0, 0, 0, 524, 549, 464, 481, 518, 501, 551, 494, 0, 0, 0, 0, 0, 0, 0, 0, 500, 499, 527, 489, 505, 505, 510, 487, 3, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4158, 4150, 4102, 4073, 4127, 4051, 4082, 4025] avg=4096.0000 max=4158 max_rank=0 min=4025 min_rank=7 imbalance_ratio=1.015137 max_load_reduction=28610 max_load_reduction_pct=0.873108 moved_to_echo=16285 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[523, 527, 537, 486, 500, 494, 516, 523, 5, 0, 0, 0, 0, 0, 0, 0, 486, 522, 536, 499, 492, 518, 483, 531, 0, 0, 0, 0, 0, 0, 0, 0, 535, 489, 508, 496, 495, 548, 506, 505, 20, 0, 0, 0, 0, 0, 0, 0, 480, 517, 532, 484, 515, 517, 529, 533, 0, 0, 0, 0, 0, 0, 0, 0, 525, 478, 545, 524, 529, 468, 484, 493, 4, 0, 0, 0, 0, 0, 0, 0, 472, 503, 492, 498, 511, 500, 508, 512, 17, 6, 0, 0, 0, 0, 0, 0, 531, 515, 518, 537, 476, 536, 541, 472, 0, 0, 0, 0, 0, 0, 0, 0, 512, 538, 517, 526, 510, 515, 542, 518, 7, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4067, 4102, 4107, 4050, 4019, 4126, 4186] avg=4096.0000 max=4186 max_rank=7 min=4019 min_rank=5 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[491, 529, 511, 507, 524, 510, 558, 495, 0, 0, 0, 0, 0, 0, 0, 0, 537, 552, 512, 496, 511, 515, 508, 463, 0, 0, 0, 0, 0, 0, 0, 0, 542, 486, 532, 520, 495, 491, 484, 500, 18, 0, 0, 0, 0, 0, 0, 0, 540, 520, 511, 498, 509, 516, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 508, 520, 522, 547, 473, 479, 524, 494, 1, 0, 0, 0, 0, 0, 0, 0, 505, 506, 479, 522, 532, 503, 556, 522, 13, 3, 0, 0, 0, 0, 0, 0, 512, 525, 537, 470, 519, 528, 522, 494, 0, 0, 0, 0, 0, 0, 0, 0, 508, 491, 513, 532, 497, 518, 517, 475, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4094, 4068, 4110, 4068, 4141, 4107, 4055] avg=4096.0000 max=4141 max_rank=5 min=4055 min_rank=7 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4096, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 3968, 4224, 4224, 4096, 4224, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[525, 529, 525, 548, 508, 540, 459, 517, 518, 535, 505, 493, 506, 506, 523, 530, 528, 492, 501, 512, 539, 493, 530, 494, 525, 520, 507, 507, 503, 541, 505, 535, 483, 476, 545, 545, 514, 528, 532, 504, 501, 513, 528, 477, 507, 546, 496, 518, 497, 518, 482, 527, 507, 500, 496, 510, 531, 489, 536, 489, 466, 520, 496, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[574, 479, 512, 527, 533, 484, 546, 514, 498, 499, 543, 522, 520, 493, 509, 505, 537, 498, 507, 525, 489, 537, 509, 497, 545, 545, 521, 480, 510, 489, 465, 515, 557, 511, 491, 527, 498, 540, 514, 517, 509, 540, 466, 503, 521, 515, 510, 525, 507, 491, 523, 489, 532, 537, 496, 500, 524, 512, 506, 477, 523, 498, 490, 492] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[482, 564, 495, 476, 513, 521, 504, 510, 510, 474, 538, 516, 504, 548, 528, 507, 462, 538, 527, 519, 516, 500, 514, 543, 516, 446, 505, 503, 545, 473, 534, 521, 536, 512, 512, 495, 511, 519, 475, 479, 552, 494, 527, 518, 547, 525, 510, 503, 515, 493, 536, 479, 497, 489, 522, 512, 557, 541, 519, 493, 523, 506, 511, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[500, 506, 521, 489, 516, 514, 516, 522, 523, 558, 502, 527, 507, 503, 512, 501, 475, 517, 501, 529, 493, 522, 537, 546, 533, 497, 506, 524, 506, 489, 537, 502, 542, 491, 518, 533, 494, 486, 525, 545, 544, 519, 495, 477, 488, 512, 509, 494, 534, 551, 522, 533, 474, 498, 480, 492, 519, 545, 477, 492, 525, 537, 504, 482] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[525, 539, 488, 487, 509, 505, 560, 503, 504, 511, 530, 506, 518, 550, 529, 502, 488, 468, 521, 534, 519, 505, 523, 504, 520, 528, 551, 508, 537, 499, 498, 546, 494, 519, 485, 476, 495, 541, 523, 500, 482, 508, 508, 515, 470, 540, 521, 514, 494, 513, 551, 517, 531, 526, 478, 498, 506, 503, 505, 529, 521, 497, 484, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[532, 521, 517, 484, 494, 510, 519, 554, 522, 495, 529, 516, 521, 499, 495, 518, 487, 505, 517, 521, 493, 506, 507, 473, 527, 499, 525, 547, 525, 519, 525, 514, 515, 539, 516, 508, 512, 537, 475, 522, 517, 524, 543, 492, 477, 517, 522, 510, 491, 510, 541, 513, 468, 526, 522, 514, 501, 470, 512, 488, 499, 517, 530, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[495, 505, 504, 502, 487, 529, 492, 547, 521, 461, 536, 538, 522, 477, 528, 540, 532, 504, 495, 521, 511, 511, 532, 505, 485, 525, 493, 475, 527, 512, 511, 518, 514, 522, 504, 497, 526, 522, 488, 504, 508, 526, 534, 507, 529, 495, 516, 491, 478, 533, 490, 532, 496, 490, 523, 516, 526, 518, 520, 532, 508, 532, 514, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[512, 538, 517, 538, 514, 496, 518, 507, 481, 468, 548, 489, 521, 511, 526, 508, 496, 516, 511, 528, 496, 494, 512, 487, 523, 510, 498, 499, 531, 551, 487, 563, 500, 500, 519, 529, 516, 529, 510, 540, 498, 482, 523, 548, 519, 500, 495, 499, 509, 528, 518, 524, 489, 479, 468, 506, 525, 498, 532, 505, 520, 529, 515, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[500, 489, 521, 489, 516, 514, 516, 522, 0, 0, 0, 0, 0, 0, 0, 0, 523, 558, 489, 527, 507, 503, 512, 501, 0, 0, 0, 0, 0, 0, 0, 0, 475, 517, 501, 529, 493, 522, 537, 546, 8, 6, 0, 0, 0, 0, 0, 0, 533, 497, 506, 524, 506, 489, 537, 484, 0, 0, 0, 0, 0, 0, 0, 0, 542, 491, 518, 533, 494, 480, 525, 545, 0, 0, 0, 0, 0, 0, 0, 0, 544, 519, 495, 477, 488, 512, 509, 494, 5, 0, 0, 0, 0, 0, 0, 0, 534, 551, 522, 533, 474, 498, 480, 492, 18, 11, 0, 0, 0, 0, 0, 0, 519, 545, 477, 492, 525, 537, 504, 482, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4120, 4134, 4076, 4128, 4043, 4113, 4087] avg=4096.0000 max=4134 max_rank=2 min=4043 min_rank=5 imbalance_ratio=1.009277 max_load_reduction=28634 max_load_reduction_pct=0.873840 moved_to_echo=16371 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 511, 525, 548, 508, 540, 459, 517, 0, 0, 0, 0, 0, 0, 0, 0, 518, 535, 492, 493, 506, 506, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 528, 492, 501, 512, 539, 493, 530, 494, 8, 6, 0, 0, 0, 0, 0, 0, 525, 520, 507, 507, 503, 541, 505, 515, 0, 0, 0, 0, 0, 0, 0, 0, 483, 476, 545, 545, 514, 521, 532, 504, 0, 0, 0, 0, 0, 0, 0, 0, 501, 513, 528, 477, 507, 546, 496, 518, 6, 0, 0, 0, 0, 0, 0, 0, 497, 518, 482, 527, 507, 500, 496, 510, 20, 12, 0, 0, 0, 0, 0, 0, 531, 489, 536, 489, 466, 520, 496, 492, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4103, 4103, 4123, 4120, 4092, 4069, 4025] avg=4096.0000 max=4133 max_rank=0 min=4025 min_rank=7 imbalance_ratio=1.009033 max_load_reduction=28635 max_load_reduction_pct=0.873871 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[482, 545, 495, 476, 513, 521, 504, 510, 0, 0, 0, 0, 0, 0, 0, 0, 510, 474, 525, 516, 504, 548, 528, 507, 0, 0, 0, 0, 0, 0, 0, 0, 462, 538, 527, 519, 516, 500, 514, 543, 8, 6, 0, 0, 0, 0, 0, 0, 516, 446, 505, 503, 545, 473, 534, 502, 0, 0, 0, 0, 0, 0, 0, 0, 536, 512, 512, 495, 511, 512, 475, 479, 0, 0, 0, 0, 0, 0, 0, 0, 552, 494, 527, 518, 547, 525, 510, 503, 6, 0, 0, 0, 0, 0, 0, 0, 515, 493, 536, 479, 497, 489, 522, 512, 19, 13, 0, 0, 0, 0, 0, 0, 557, 541, 519, 493, 523, 506, 511, 508, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4112, 4133, 4024, 4032, 4182, 4075, 4164] avg=4096.0000 max=4182 max_rank=5 min=4024 min_rank=3 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16453 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[574, 463, 512, 527, 533, 484, 546, 514, 0, 0, 0, 0, 0, 0, 0, 0, 498, 499, 530, 522, 520, 493, 509, 505, 0, 0, 0, 0, 0, 0, 0, 0, 537, 498, 507, 525, 489, 537, 509, 497, 8, 5, 0, 0, 0, 0, 0, 0, 545, 545, 521, 480, 510, 489, 465, 496, 0, 0, 0, 0, 0, 0, 0, 0, 557, 511, 491, 527, 498, 533, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 509, 540, 466, 503, 521, 515, 510, 525, 6, 0, 0, 0, 0, 0, 0, 0, 507, 491, 523, 489, 532, 537, 496, 500, 19, 11, 0, 0, 0, 0, 0, 0, 524, 512, 506, 477, 523, 498, 490, 492, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4153, 4076, 4112, 4051, 4148, 4095, 4105, 4028] avg=4096.0000 max=4153 max_rank=0 min=4028 min_rank=7 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 521, 488, 487, 509, 505, 560, 503, 0, 0, 0, 0, 0, 0, 0, 0, 504, 511, 517, 506, 518, 550, 529, 502, 0, 0, 0, 0, 0, 0, 0, 0, 488, 468, 521, 534, 519, 505, 523, 504, 8, 6, 0, 0, 0, 0, 0, 0, 520, 528, 551, 508, 537, 499, 498, 526, 0, 0, 0, 0, 0, 0, 0, 0, 494, 519, 485, 476, 495, 534, 523, 500, 0, 0, 0, 0, 0, 0, 0, 0, 482, 508, 508, 515, 470, 540, 521, 514, 6, 0, 0, 0, 0, 0, 0, 0, 494, 513, 551, 517, 531, 526, 478, 498, 20, 12, 0, 0, 0, 0, 0, 0, 506, 503, 505, 529, 521, 497, 484, 509, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4098, 4137, 4076, 4167, 4026, 4064, 4140, 4060] avg=4096.0000 max=4167 max_rank=3 min=4026 min_rank=4 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16290 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[495, 488, 504, 502, 487, 529, 492, 547, 0, 0, 0, 0, 0, 0, 0, 0, 521, 461, 523, 538, 522, 477, 528, 540, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 495, 521, 511, 511, 532, 505, 8, 6, 0, 0, 0, 0, 0, 0, 485, 525, 493, 475, 527, 512, 511, 499, 0, 0, 0, 0, 0, 0, 0, 0, 514, 522, 504, 497, 526, 515, 488, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 534, 507, 529, 495, 516, 491, 6, 0, 0, 0, 0, 0, 0, 0, 478, 533, 490, 532, 496, 490, 523, 516, 19, 11, 0, 0, 0, 0, 0, 0, 526, 518, 520, 532, 508, 532, 514, 536, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4110, 4125, 4027, 4070, 4112, 4088, 4192] avg=4096.0000 max=4192 max_rank=7 min=4027 min_rank=3 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[532, 495, 517, 484, 494, 510, 519, 554, 0, 0, 0, 0, 0, 0, 0, 0, 522, 495, 505, 516, 521, 499, 495, 518, 0, 0, 0, 0, 0, 0, 0, 0, 487, 505, 517, 521, 493, 506, 507, 473, 13, 9, 0, 0, 0, 0, 0, 0, 527, 499, 525, 547, 525, 519, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 515, 539, 516, 508, 512, 521, 475, 522, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 543, 492, 477, 517, 522, 510, 8, 0, 0, 0, 0, 0, 0, 0, 491, 510, 541, 513, 468, 526, 522, 514, 22, 17, 0, 0, 0, 0, 0, 0, 501, 470, 512, 488, 499, 517, 530, 524, 11, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4071, 4031, 4159, 4108, 4110, 4124, 4060] avg=4096.0000 max=4159 max_rank=3 min=4031 min_rank=2 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[512, 520, 517, 538, 514, 496, 518, 507, 0, 0, 0, 0, 0, 0, 0, 0, 481, 468, 535, 489, 521, 511, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 496, 516, 511, 528, 496, 494, 512, 487, 8, 6, 0, 0, 0, 0, 0, 0, 523, 510, 498, 499, 531, 551, 487, 542, 0, 0, 0, 0, 0, 0, 0, 0, 500, 500, 519, 529, 516, 522, 510, 540, 0, 0, 0, 0, 0, 0, 0, 0, 498, 482, 523, 548, 519, 500, 495, 499, 6, 0, 0, 0, 0, 0, 0, 0, 509, 528, 518, 524, 489, 479, 468, 506, 21, 12, 0, 0, 0, 0, 0, 0, 525, 498, 532, 505, 520, 529, 515, 522, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4039, 4054, 4141, 4136, 4070, 4054, 4152] avg=4096.0000 max=4152 max_rank=7 min=4039 min_rank=1 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16412 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[532, 512, 527, 505, 528, 476, 531, 521, 531, 545, 521, 525, 515, 496, 499, 476, 541, 499, 478, 562, 494, 507, 499, 523, 480, 539, 484, 520, 521, 500, 517, 492, 526, 529, 490, 494, 491, 508, 509, 524, 507, 504, 525, 507, 518, 500, 532, 509, 533, 541, 520, 520, 489, 481, 524, 526, 480, 496, 531, 537, 516, 485, 499, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[511, 494, 494, 521, 542, 541, 491, 542, 518, 525, 525, 490, 507, 515, 514, 525, 546, 513, 531, 478, 495, 524, 506, 501, 519, 508, 537, 526, 500, 514, 499, 512, 511, 519, 545, 515, 544, 517, 513, 510, 467, 544, 504, 484, 502, 489, 487, 514, 514, 495, 509, 501, 520, 480, 512, 500, 515, 490, 481, 516, 538, 517, 536, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[516, 519, 533, 484, 541, 533, 532, 527, 531, 509, 501, 501, 549, 489, 481, 529, 511, 502, 486, 515, 521, 487, 500, 490, 494, 477, 535, 518, 507, 533, 461, 521, 498, 533, 505, 538, 471, 481, 507, 494, 512, 543, 531, 507, 483, 529, 479, 500, 546, 486, 522, 526, 500, 508, 534, 504, 521, 527, 526, 500, 533, 529, 514, 548] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[519, 525, 528, 500, 527, 509, 520, 506, 501, 511, 511, 514, 500, 517, 532, 527, 537, 499, 513, 494, 533, 480, 487, 509, 491, 537, 528, 477, 487, 554, 520, 478, 536, 514, 464, 467, 523, 502, 516, 510, 527, 518, 514, 540, 468, 540, 523, 532, 487, 552, 541, 521, 498, 529, 517, 501, 528, 499, 517, 479, 486, 547, 506, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[533, 509, 504, 503, 484, 514, 496, 516, 479, 490, 492, 509, 503, 555, 523, 536, 522, 483, 493, 466, 517, 532, 522, 504, 506, 511, 527, 531, 526, 510, 528, 503, 538, 530, 522, 471, 475, 512, 510, 494, 534, 479, 519, 521, 518, 472, 544, 519, 502, 516, 565, 516, 526, 535, 513, 503, 509, 474, 535, 496, 548, 528, 503, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[493, 521, 544, 510, 498, 527, 471, 505, 495, 540, 510, 493, 503, 526, 516, 497, 508, 472, 537, 532, 526, 533, 537, 496, 528, 519, 499, 501, 541, 502, 540, 503, 506, 509, 526, 521, 540, 500, 525, 511, 540, 524, 520, 521, 453, 472, 508, 488, 517, 521, 507, 517, 496, 508, 530, 517, 509, 474, 544, 501, 497, 519, 495, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[484, 500, 515, 518, 526, 492, 511, 508, 476, 543, 520, 504, 528, 515, 504, 549, 518, 500, 536, 485, 520, 532, 496, 504, 492, 528, 534, 505, 525, 490, 509, 516, 519, 544, 514, 517, 545, 530, 519, 514, 479, 497, 521, 527, 503, 505, 498, 487, 505, 455, 527, 502, 510, 561, 508, 493, 505, 515, 540, 514, 471, 503, 528, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[555, 521, 545, 553, 510, 501, 513, 497, 539, 472, 519, 489, 498, 527, 525, 522, 529, 526, 504, 526, 516, 501, 518, 515, 524, 470, 485, 529, 515, 523, 545, 543, 501, 512, 498, 520, 516, 496, 519, 492, 502, 491, 530, 527, 488, 502, 518, 499, 515, 503, 529, 524, 513, 536, 444, 480, 522, 521, 494, 468, 478, 571, 500, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[533, 509, 481, 503, 484, 514, 496, 516, 0, 0, 0, 0, 0, 0, 0, 0, 479, 490, 492, 509, 503, 555, 523, 529, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 493, 466, 517, 532, 522, 504, 7, 4, 0, 0, 0, 0, 0, 0, 506, 511, 525, 531, 526, 510, 528, 503, 0, 0, 0, 0, 0, 0, 0, 0, 538, 530, 522, 471, 475, 512, 510, 494, 2, 0, 0, 0, 0, 0, 0, 0, 534, 479, 519, 521, 518, 472, 544, 519, 23, 3, 0, 0, 0, 0, 0, 0, 502, 516, 558, 516, 526, 535, 513, 503, 0, 0, 0, 0, 0, 0, 0, 0, 509, 474, 535, 496, 548, 528, 503, 514, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4080, 4050, 4140, 4054, 4132, 4169, 4107] avg=4096.0000 max=4169 max_rank=6 min=4036 min_rank=0 imbalance_ratio=1.017822 max_load_reduction=28599 max_load_reduction_pct=0.872772 moved_to_echo=16462 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 519, 509, 484, 541, 533, 532, 527, 0, 0, 0, 0, 0, 0, 0, 0, 531, 509, 501, 501, 549, 489, 481, 522, 0, 0, 0, 0, 0, 0, 0, 0, 511, 502, 486, 515, 521, 487, 500, 490, 7, 4, 0, 0, 0, 0, 0, 0, 494, 477, 533, 518, 507, 533, 461, 521, 0, 0, 0, 0, 0, 0, 0, 0, 498, 533, 505, 538, 471, 481, 507, 494, 2, 0, 0, 0, 0, 0, 0, 0, 512, 543, 531, 507, 483, 529, 479, 500, 24, 3, 0, 0, 0, 0, 0, 0, 546, 486, 515, 526, 500, 508, 534, 504, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 526, 500, 533, 529, 514, 548, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4161, 4083, 4023, 4044, 4029, 4111, 4119, 4198] avg=4096.0000 max=4198 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16457 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[532, 512, 503, 505, 528, 476, 531, 521, 0, 0, 0, 0, 0, 0, 0, 0, 531, 545, 521, 525, 515, 496, 499, 470, 0, 0, 0, 0, 0, 0, 0, 0, 541, 499, 478, 562, 494, 507, 499, 523, 6, 4, 0, 0, 0, 0, 0, 0, 480, 539, 482, 520, 521, 500, 517, 492, 0, 0, 0, 0, 0, 0, 0, 0, 526, 529, 490, 494, 491, 508, 509, 524, 2, 0, 0, 0, 0, 0, 0, 0, 507, 504, 525, 507, 518, 500, 532, 509, 24, 3, 0, 0, 0, 0, 0, 0, 533, 541, 513, 520, 489, 481, 524, 526, 0, 0, 0, 0, 0, 0, 0, 0, 480, 496, 531, 537, 516, 485, 499, 521, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4108, 4102, 4113, 4051, 4073, 4129, 4127, 4065] avg=4096.0000 max=4129 max_rank=5 min=4051 min_rank=3 imbalance_ratio=1.008057 max_load_reduction=28639 max_load_reduction_pct=0.873993 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 494, 471, 521, 542, 541, 491, 542, 0, 0, 0, 0, 0, 0, 0, 0, 518, 525, 525, 490, 507, 515, 514, 518, 0, 0, 0, 0, 0, 0, 0, 0, 546, 513, 531, 478, 495, 524, 506, 501, 7, 4, 0, 0, 0, 0, 0, 0, 519, 508, 535, 526, 500, 514, 499, 512, 0, 0, 0, 0, 0, 0, 0, 0, 511, 519, 545, 515, 544, 517, 513, 510, 2, 0, 0, 0, 0, 0, 0, 0, 467, 544, 504, 484, 502, 489, 487, 514, 23, 3, 0, 0, 0, 0, 0, 0, 514, 495, 502, 501, 520, 480, 512, 500, 0, 0, 0, 0, 0, 0, 0, 0, 515, 490, 481, 516, 538, 517, 536, 515, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4113, 4112, 4105, 4113, 4176, 4017, 4024, 4108] avg=4096.0000 max=4176 max_rank=4 min=4017 min_rank=5 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16325 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[484, 500, 491, 518, 526, 492, 511, 508, 0, 0, 0, 0, 0, 0, 0, 0, 476, 543, 520, 504, 528, 515, 504, 542, 0, 0, 0, 0, 0, 0, 0, 0, 518, 500, 536, 485, 520, 532, 496, 504, 7, 4, 0, 0, 0, 0, 0, 0, 492, 528, 532, 505, 525, 490, 509, 516, 0, 0, 0, 0, 0, 0, 0, 0, 519, 544, 514, 517, 545, 530, 519, 514, 2, 0, 0, 0, 0, 0, 0, 0, 479, 497, 521, 527, 503, 505, 498, 487, 24, 3, 0, 0, 0, 0, 0, 0, 505, 455, 520, 502, 510, 561, 508, 493, 0, 0, 0, 0, 0, 0, 0, 0, 505, 515, 540, 514, 471, 503, 528, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4030, 4132, 4102, 4097, 4204, 4044, 4054, 4105] avg=4096.0000 max=4204 max_rank=4 min=4030 min_rank=0 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[555, 521, 520, 553, 510, 501, 513, 497, 0, 0, 0, 0, 0, 0, 0, 0, 539, 472, 519, 489, 498, 527, 525, 515, 0, 0, 0, 0, 0, 0, 0, 0, 529, 526, 504, 526, 516, 501, 518, 515, 7, 4, 0, 0, 0, 0, 0, 0, 524, 470, 483, 529, 515, 523, 545, 543, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 498, 520, 516, 496, 519, 492, 2, 0, 0, 0, 0, 0, 0, 0, 502, 491, 530, 527, 488, 502, 518, 499, 25, 3, 0, 0, 0, 0, 0, 0, 515, 503, 522, 524, 513, 536, 444, 480, 0, 0, 0, 0, 0, 0, 0, 0, 522, 521, 494, 468, 478, 571, 500, 504, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4170, 4084, 4146, 4132, 4056, 4085, 4037, 4058] avg=4096.0000 max=4170 max_rank=0 min=4037 min_rank=6 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16236 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[519, 525, 500, 500, 527, 509, 520, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 511, 511, 514, 500, 517, 532, 516, 0, 0, 0, 0, 0, 0, 0, 0, 537, 499, 513, 494, 533, 480, 487, 509, 11, 8, 6, 0, 0, 0, 0, 0, 491, 537, 516, 477, 487, 554, 520, 478, 0, 0, 0, 0, 0, 0, 0, 0, 536, 514, 464, 467, 523, 502, 516, 510, 4, 0, 0, 0, 0, 0, 0, 0, 527, 518, 514, 540, 468, 540, 523, 532, 28, 6, 0, 0, 0, 0, 0, 0, 487, 552, 527, 521, 498, 529, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 528, 499, 517, 479, 486, 547, 506, 495, 2, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4106, 4102, 4077, 4060, 4036, 4196, 4132, 4059] avg=4096.0000 max=4196 max_rank=5 min=4036 min_rank=4 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16423 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[493, 521, 519, 510, 498, 527, 471, 505, 0, 0, 0, 0, 0, 0, 0, 0, 495, 540, 510, 493, 503, 526, 516, 490, 0, 0, 0, 0, 0, 0, 0, 0, 508, 472, 537, 532, 526, 533, 537, 496, 7, 4, 0, 0, 0, 0, 0, 0, 528, 519, 497, 501, 541, 502, 540, 503, 0, 0, 0, 0, 0, 0, 0, 0, 506, 509, 526, 521, 540, 500, 525, 511, 2, 0, 0, 0, 0, 0, 0, 0, 540, 524, 520, 521, 453, 472, 508, 488, 25, 3, 0, 0, 0, 0, 0, 0, 517, 521, 500, 517, 496, 508, 530, 517, 0, 0, 0, 0, 0, 0, 0, 0, 509, 474, 544, 501, 497, 519, 495, 529, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4073, 4152, 4131, 4140, 4054, 4106, 4068] avg=4096.0000 max=4152 max_rank=2 min=4044 min_rank=0 imbalance_ratio=1.013672 max_load_reduction=28616 max_load_reduction_pct=0.873291 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[497, 505, 539, 529, 522, 507, 489, 492, 494, 541, 518, 494, 516, 506, 516, 511, 495, 512, 520, 532, 563, 547, 490, 534, 475, 525, 522, 526, 480, 507, 525, 486, 495, 501, 525, 516, 532, 533, 500, 488, 497, 506, 505, 520, 479, 507, 524, 502, 536, 545, 502, 468, 469, 498, 563, 530, 500, 528, 509, 512, 534, 506, 506, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[508, 560, 537, 594, 505, 525, 511, 529, 521, 481, 548, 475, 519, 517, 497, 554, 498, 529, 512, 562, 503, 537, 500, 520, 532, 506, 482, 489, 515, 511, 516, 526, 524, 476, 535, 522, 520, 523, 504, 489, 471, 500, 502, 519, 506, 514, 512, 542, 506, 473, 473, 484, 484, 489, 504, 476, 533, 472, 492, 527, 526, 518, 537, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[542, 522, 552, 493, 519, 466, 523, 499, 471, 515, 507, 519, 493, 545, 512, 555, 518, 506, 485, 494, 507, 508, 492, 514, 536, 504, 540, 536, 523, 551, 499, 515, 513, 518, 482, 546, 495, 455, 553, 525, 535, 516, 529, 502, 513, 526, 500, 510, 503, 510, 506, 508, 518, 561, 505, 500, 514, 513, 464, 517, 493, 473, 524, 480] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[529, 512, 548, 500, 530, 542, 503, 499, 548, 548, 536, 500, 491, 517, 523, 500, 493, 499, 545, 516, 506, 478, 518, 511, 528, 506, 514, 473, 514, 512, 510, 506, 547, 535, 508, 529, 505, 496, 515, 509, 523, 505, 476, 551, 522, 530, 531, 502, 466, 494, 498, 526, 472, 515, 489, 507, 510, 521, 527, 550, 524, 499, 464, 467] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[530, 487, 518, 540, 554, 478, 546, 523, 510, 516, 501, 520, 518, 524, 508, 515, 506, 515, 521, 490, 515, 452, 519, 512, 511, 491, 486, 548, 526, 504, 527, 484, 485, 550, 516, 502, 523, 520, 559, 519, 473, 508, 513, 489, 497, 518, 527, 516, 524, 509, 523, 468, 509, 513, 489, 511, 481, 520, 523, 530, 519, 525, 493, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[508, 512, 520, 508, 515, 492, 511, 504, 508, 472, 520, 511, 513, 483, 500, 509, 478, 478, 557, 526, 523, 520, 552, 506, 527, 553, 499, 465, 547, 520, 504, 521, 507, 531, 473, 533, 497, 500, 508, 488, 502, 520, 520, 478, 520, 533, 491, 504, 470, 508, 506, 523, 524, 529, 519, 522, 468, 509, 540, 556, 544, 548, 510, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[515, 527, 534, 543, 490, 528, 522, 560, 558, 527, 477, 484, 506, 502, 474, 471, 514, 529, 541, 529, 479, 511, 488, 502, 499, 549, 539, 516, 525, 527, 505, 478, 523, 493, 501, 521, 523, 478, 482, 499, 478, 533, 484, 549, 517, 517, 490, 479, 500, 543, 520, 527, 504, 478, 522, 516, 513, 541, 554, 490, 498, 492, 523, 531] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[498, 511, 514, 510, 529, 523, 537, 511, 507, 487, 533, 484, 541, 502, 512, 508, 517, 524, 514, 498, 541, 497, 517, 492, 530, 524, 516, 564, 499, 486, 508, 484, 525, 514, 523, 538, 569, 511, 535, 513, 535, 523, 503, 453, 532, 508, 512, 487, 534, 479, 499, 484, 469, 556, 552, 482, 528, 514, 519, 493, 493, 508, 484, 475] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[497, 505, 482, 529, 522, 507, 489, 492, 0, 0, 0, 0, 0, 0, 0, 0, 494, 541, 518, 494, 516, 506, 516, 511, 9, 0, 0, 0, 0, 0, 0, 0, 495, 512, 516, 532, 563, 547, 490, 534, 0, 0, 0, 0, 0, 0, 0, 0, 475, 515, 522, 526, 480, 507, 525, 486, 0, 0, 0, 0, 0, 0, 0, 0, 495, 501, 525, 504, 532, 533, 500, 488, 0, 0, 0, 0, 0, 0, 0, 0, 497, 506, 505, 520, 479, 507, 524, 502, 12, 10, 0, 0, 0, 0, 0, 0, 536, 545, 502, 468, 469, 498, 563, 530, 47, 0, 0, 0, 0, 0, 0, 0, 500, 528, 509, 512, 534, 506, 506, 517, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4105, 4189, 4036, 4078, 4062, 4158, 4117] avg=4096.0000 max=4189 max_rank=2 min=4023 min_rank=0 imbalance_ratio=1.022705 max_load_reduction=28579 max_load_reduction_pct=0.872162 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 560, 480, 594, 505, 525, 511, 529, 0, 0, 0, 0, 0, 0, 0, 0, 521, 481, 548, 475, 519, 517, 497, 554, 9, 0, 0, 0, 0, 0, 0, 0, 498, 529, 508, 562, 503, 537, 500, 520, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 482, 489, 515, 511, 516, 526, 0, 0, 0, 0, 0, 0, 0, 0, 524, 476, 535, 510, 520, 523, 504, 489, 0, 0, 0, 0, 0, 0, 0, 0, 471, 500, 502, 519, 506, 514, 512, 542, 12, 10, 0, 0, 0, 0, 0, 0, 506, 473, 473, 484, 484, 489, 504, 476, 47, 0, 0, 0, 0, 0, 0, 0, 533, 472, 492, 527, 526, 518, 537, 496, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4121, 4157, 4068, 4081, 4088, 3936, 4105] avg=4096.0000 max=4212 max_rank=0 min=3936 min_rank=6 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16210 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[529, 512, 490, 500, 530, 542, 503, 499, 0, 0, 0, 0, 0, 0, 0, 0, 548, 548, 536, 500, 491, 517, 523, 500, 9, 0, 0, 0, 0, 0, 0, 0, 493, 499, 540, 516, 506, 478, 518, 511, 0, 0, 0, 0, 0, 0, 0, 0, 528, 497, 514, 473, 514, 512, 510, 506, 0, 0, 0, 0, 0, 0, 0, 0, 547, 535, 508, 517, 505, 496, 515, 509, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 476, 551, 522, 530, 531, 502, 12, 10, 0, 0, 0, 0, 0, 0, 466, 494, 498, 526, 472, 515, 489, 507, 48, 0, 0, 0, 0, 0, 0, 0, 510, 521, 527, 550, 524, 499, 464, 467, 5, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4172, 4061, 4054, 4132, 4162, 4015, 4067] avg=4096.0000 max=4172 max_rank=1 min=4015 min_rank=6 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[530, 487, 464, 540, 554, 478, 546, 523, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 501, 520, 518, 524, 508, 515, 8, 0, 0, 0, 0, 0, 0, 0, 506, 515, 517, 490, 515, 452, 519, 512, 0, 0, 0, 0, 0, 0, 0, 0, 511, 483, 486, 548, 526, 504, 527, 484, 0, 0, 0, 0, 0, 0, 0, 0, 485, 550, 516, 490, 523, 520, 559, 519, 0, 0, 0, 0, 0, 0, 0, 0, 473, 508, 513, 489, 497, 518, 527, 516, 12, 9, 0, 0, 0, 0, 0, 0, 524, 509, 523, 468, 509, 513, 489, 511, 45, 0, 0, 0, 0, 0, 0, 0, 481, 520, 523, 530, 519, 525, 493, 521, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4122, 4120, 4026, 4069, 4162, 4062, 4091, 4116] avg=4096.0000 max=4162 max_rank=4 min=4026 min_rank=2 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[542, 522, 494, 493, 519, 466, 523, 499, 0, 0, 0, 0, 0, 0, 0, 0, 471, 515, 507, 519, 493, 545, 512, 555, 9, 0, 0, 0, 0, 0, 0, 0, 518, 506, 481, 494, 507, 508, 492, 514, 0, 0, 0, 0, 0, 0, 0, 0, 536, 495, 540, 536, 523, 551, 499, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 518, 482, 533, 495, 455, 553, 525, 0, 0, 0, 0, 0, 0, 0, 0, 535, 516, 529, 502, 513, 526, 500, 510, 13, 10, 0, 0, 0, 0, 0, 0, 503, 510, 506, 508, 518, 561, 505, 500, 48, 0, 0, 0, 0, 0, 0, 0, 514, 513, 464, 517, 493, 473, 524, 480, 4, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4126, 4020, 4195, 4074, 4154, 4159, 3982] avg=4096.0000 max=4195 max_rank=3 min=3982 min_rank=7 imbalance_ratio=1.024170 max_load_reduction=28573 max_load_reduction_pct=0.871979 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[508, 512, 465, 508, 515, 492, 511, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 472, 520, 511, 513, 483, 500, 509, 10, 0, 0, 0, 0, 0, 0, 0, 478, 478, 552, 526, 523, 520, 552, 506, 0, 0, 0, 0, 0, 0, 0, 0, 527, 542, 499, 465, 547, 520, 504, 521, 0, 0, 0, 0, 0, 0, 0, 0, 507, 531, 473, 521, 497, 500, 508, 488, 0, 0, 0, 0, 0, 0, 0, 0, 502, 520, 520, 478, 520, 533, 491, 504, 12, 9, 0, 0, 0, 0, 0, 0, 470, 508, 506, 523, 524, 529, 519, 522, 46, 0, 0, 0, 0, 0, 0, 0, 468, 509, 540, 556, 544, 548, 510, 525, 5, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4015, 4026, 4135, 4125, 4025, 4089, 4147, 4206] avg=4096.0000 max=4206 max_rank=7 min=4015 min_rank=0 imbalance_ratio=1.026855 max_load_reduction=28562 max_load_reduction_pct=0.871643 moved_to_echo=16467 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[515, 527, 469, 543, 490, 528, 522, 560, 0, 0, 0, 0, 0, 0, 0, 0, 558, 527, 477, 484, 506, 502, 474, 471, 13, 3, 0, 0, 0, 0, 0, 0, 514, 529, 532, 529, 479, 511, 488, 502, 0, 0, 0, 0, 0, 0, 0, 0, 499, 531, 539, 516, 525, 527, 505, 478, 0, 0, 0, 0, 0, 0, 0, 0, 523, 493, 501, 502, 523, 478, 482, 499, 0, 0, 0, 0, 0, 0, 0, 0, 478, 533, 484, 549, 517, 517, 490, 479, 16, 13, 0, 0, 0, 0, 0, 0, 500, 543, 520, 527, 504, 478, 522, 516, 52, 0, 0, 0, 0, 0, 0, 0, 513, 541, 554, 490, 498, 492, 523, 531, 9, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4154, 4015, 4084, 4120, 4001, 4076, 4162, 4156] avg=4096.0000 max=4162 max_rank=6 min=4001 min_rank=4 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[498, 511, 460, 510, 529, 523, 537, 511, 0, 0, 0, 0, 0, 0, 0, 0, 507, 487, 533, 484, 541, 502, 512, 508, 9, 0, 0, 0, 0, 0, 0, 0, 517, 524, 510, 498, 541, 497, 517, 492, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 516, 564, 499, 486, 508, 484, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 523, 525, 569, 511, 535, 513, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 503, 453, 532, 508, 512, 487, 13, 9, 0, 0, 0, 0, 0, 0, 534, 479, 499, 484, 469, 556, 552, 482, 45, 0, 0, 0, 0, 0, 0, 0, 528, 514, 519, 493, 493, 508, 484, 475, 4, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4083, 4096, 4101, 4215, 4075, 4100, 4019] avg=4096.0000 max=4215 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.029053 max_load_reduction=28553 max_load_reduction_pct=0.871368 moved_to_echo=16409 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 3840, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 3968, 4224, 4224, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[508, 525, 515, 503, 528, 520, 500, 539, 534, 498, 472, 495, 519, 522, 538, 534, 566, 484, 501, 504, 517, 510, 515, 508, 522, 512, 498, 484, 521, 535, 516, 532, 531, 508, 532, 508, 519, 536, 479, 501, 513, 543, 535, 512, 481, 517, 473, 510, 503, 508, 494, 533, 494, 506, 509, 523, 519, 488, 484, 506, 513, 485, 519, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[494, 507, 537, 511, 480, 509, 486, 490, 523, 517, 521, 520, 542, 513, 529, 496, 508, 500, 533, 484, 485, 506, 554, 518, 498, 525, 529, 515, 508, 503, 516, 521, 513, 487, 542, 528, 500, 503, 496, 504, 520, 542, 519, 535, 519, 520, 519, 463, 500, 477, 492, 489, 551, 505, 479, 518, 547, 515, 500, 505, 548, 554, 495, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[483, 553, 493, 489, 514, 478, 499, 530, 481, 518, 534, 576, 497, 514, 515, 481, 584, 489, 541, 548, 557, 529, 534, 491, 519, 499, 491, 513, 535, 530, 515, 527, 535, 523, 489, 507, 543, 509, 498, 497, 490, 475, 481, 519, 527, 507, 508, 500, 517, 482, 517, 528, 497, 492, 506, 514, 521, 485, 491, 525, 533, 483, 523, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[529, 518, 496, 479, 504, 532, 492, 522, 521, 513, 507, 500, 514, 525, 478, 511, 537, 532, 478, 583, 530, 479, 516, 508, 516, 506, 544, 522, 500, 486, 519, 498, 486, 509, 530, 558, 511, 517, 492, 499, 567, 501, 538, 510, 516, 498, 491, 469, 540, 513, 514, 506, 519, 524, 494, 494, 508, 487, 502, 534, 489, 512, 507, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[520, 500, 495, 522, 539, 483, 514, 523, 531, 535, 516, 520, 496, 531, 513, 513, 543, 518, 499, 486, 491, 542, 521, 504, 512, 536, 519, 531, 509, 509, 514, 501, 526, 447, 549, 505, 539, 510, 504, 530, 531, 500, 509, 549, 502, 501, 511, 487, 512, 495, 497, 482, 499, 483, 463, 512, 525, 531, 554, 501, 553, 504, 466, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[509, 516, 488, 524, 515, 511, 532, 506, 529, 517, 542, 532, 483, 520, 553, 524, 518, 500, 513, 465, 509, 466, 528, 525, 513, 521, 525, 518, 498, 535, 507, 525, 515, 513, 517, 510, 520, 537, 538, 533, 521, 507, 531, 514, 522, 507, 516, 496, 498, 543, 497, 498, 487, 489, 512, 512, 495, 499, 497, 499, 478, 529, 469, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[497, 542, 528, 542, 549, 491, 473, 549, 528, 512, 526, 476, 489, 499, 519, 512, 545, 517, 487, 493, 515, 528, 475, 541, 500, 484, 481, 534, 529, 538, 503, 449, 484, 498, 527, 500, 515, 515, 544, 516, 534, 534, 522, 514, 493, 509, 506, 539, 503, 529, 545, 482, 507, 549, 465, 520, 505, 526, 565, 485, 514, 495, 512, 465] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[578, 518, 513, 528, 525, 525, 525, 532, 521, 495, 484, 482, 540, 518, 537, 540, 505, 512, 516, 501, 504, 506, 479, 519, 522, 476, 494, 511, 528, 491, 527, 493, 535, 482, 508, 545, 549, 526, 510, 518, 532, 495, 523, 528, 535, 510, 499, 489, 460, 507, 537, 503, 485, 480, 505, 484, 509, 535, 509, 497, 479, 506, 548, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[483, 553, 493, 489, 514, 478, 499, 517, 0, 0, 0, 0, 0, 0, 0, 0, 481, 518, 534, 576, 497, 514, 488, 481, 0, 0, 0, 0, 0, 0, 0, 0, 567, 489, 541, 548, 557, 529, 534, 491, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 491, 513, 533, 530, 515, 527, 0, 0, 0, 0, 0, 0, 0, 0, 535, 523, 489, 507, 519, 509, 498, 497, 0, 0, 0, 0, 0, 0, 0, 0, 489, 475, 481, 519, 527, 507, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 517, 482, 517, 528, 497, 492, 506, 514, 27, 24, 10, 0, 0, 0, 0, 0, 521, 485, 491, 525, 533, 483, 523, 489, 13, 7, 2, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4026, 4089, 4256, 4127, 4077, 4006, 4114, 4073] avg=4096.0000 max=4256 max_rank=2 min=4006 min_rank=5 imbalance_ratio=1.039062 max_load_reduction=28512 max_load_reduction_pct=0.870117 moved_to_echo=16270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[494, 507, 537, 511, 480, 509, 486, 478, 0, 0, 0, 0, 0, 0, 0, 0, 523, 517, 521, 520, 542, 513, 501, 496, 0, 0, 0, 0, 0, 0, 0, 0, 493, 500, 533, 484, 485, 506, 554, 518, 0, 0, 0, 0, 0, 0, 0, 0, 498, 525, 529, 515, 506, 503, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 513, 487, 542, 528, 478, 503, 496, 504, 0, 0, 0, 0, 0, 0, 0, 0, 519, 542, 519, 535, 519, 520, 519, 463, 0, 0, 0, 0, 0, 0, 0, 0, 500, 477, 492, 489, 551, 505, 479, 518, 28, 22, 9, 0, 0, 0, 0, 0, 547, 515, 500, 505, 548, 554, 495, 505, 12, 6, 2, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4002, 4133, 4073, 4113, 4051, 4136, 4070, 4190] avg=4096.0000 max=4190 max_rank=7 min=4002 min_rank=0 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16447 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[529, 518, 496, 479, 504, 532, 492, 509, 0, 0, 0, 0, 0, 0, 0, 0, 521, 513, 507, 500, 514, 525, 453, 511, 0, 0, 0, 0, 0, 0, 0, 0, 522, 532, 478, 583, 530, 479, 516, 508, 0, 0, 0, 0, 0, 0, 0, 0, 516, 506, 544, 522, 498, 486, 519, 498, 0, 0, 0, 0, 0, 0, 0, 0, 486, 509, 530, 558, 489, 517, 492, 499, 0, 0, 0, 0, 0, 0, 0, 0, 565, 501, 538, 510, 516, 498, 491, 469, 0, 0, 0, 0, 0, 0, 0, 0, 540, 513, 514, 506, 519, 524, 494, 494, 25, 22, 9, 0, 0, 0, 0, 0, 508, 487, 502, 534, 489, 512, 507, 538, 13, 6, 2, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4059, 4044, 4148, 4089, 4080, 4088, 4160, 4100] avg=4096.0000 max=4160 max_rank=6 min=4044 min_rank=1 imbalance_ratio=1.015625 max_load_reduction=28608 max_load_reduction_pct=0.873047 moved_to_echo=16428 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[508, 525, 515, 503, 528, 520, 500, 526, 0, 0, 0, 0, 0, 0, 0, 0, 534, 498, 472, 495, 519, 522, 510, 534, 0, 0, 0, 0, 0, 0, 0, 0, 550, 484, 501, 504, 517, 510, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 522, 512, 498, 484, 519, 535, 516, 532, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 532, 508, 496, 536, 479, 501, 0, 0, 0, 0, 0, 0, 0, 0, 512, 543, 535, 512, 481, 517, 473, 510, 0, 0, 0, 0, 0, 0, 0, 0, 503, 508, 494, 533, 494, 506, 509, 523, 28, 23, 10, 0, 0, 0, 0, 0, 519, 488, 484, 506, 513, 485, 519, 511, 13, 6, 2, 1, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4084, 4089, 4118, 4091, 4083, 4131, 4047] avg=4096.0000 max=4131 max_rank=6 min=4047 min_rank=7 imbalance_ratio=1.008545 max_load_reduction=28637 max_load_reduction_pct=0.873932 moved_to_echo=16352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[497, 542, 528, 542, 549, 491, 473, 535, 0, 0, 0, 0, 0, 0, 0, 0, 528, 512, 526, 476, 489, 499, 492, 512, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 487, 493, 515, 528, 475, 541, 0, 0, 0, 0, 0, 0, 0, 0, 500, 484, 481, 534, 527, 538, 503, 449, 0, 0, 0, 0, 0, 0, 0, 0, 484, 498, 527, 500, 493, 515, 544, 516, 0, 0, 0, 0, 0, 0, 0, 0, 532, 534, 522, 514, 493, 509, 506, 539, 0, 0, 0, 0, 0, 0, 0, 0, 503, 529, 545, 482, 507, 549, 465, 520, 27, 22, 10, 0, 0, 0, 0, 0, 505, 526, 565, 485, 514, 495, 512, 465, 14, 6, 2, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4157, 4034, 4085, 4016, 4077, 4149, 4159, 4091] avg=4096.0000 max=4159 max_rank=6 min=4016 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[520, 500, 495, 522, 539, 483, 514, 510, 0, 0, 0, 0, 0, 0, 0, 0, 531, 535, 516, 520, 496, 531, 486, 513, 0, 0, 0, 0, 0, 0, 0, 0, 527, 518, 499, 486, 491, 542, 521, 504, 0, 0, 0, 0, 0, 0, 0, 0, 512, 536, 519, 531, 507, 509, 514, 501, 0, 0, 0, 0, 0, 0, 0, 0, 526, 447, 549, 505, 515, 510, 504, 530, 0, 0, 0, 0, 0, 0, 0, 0, 529, 500, 509, 549, 502, 501, 511, 487, 0, 0, 0, 0, 0, 0, 0, 0, 512, 495, 497, 482, 499, 483, 463, 512, 27, 24, 10, 0, 0, 0, 0, 0, 525, 531, 554, 501, 553, 504, 466, 505, 13, 6, 2, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4128, 4088, 4129, 4086, 4088, 4004, 4162] avg=4096.0000 max=4162 max_rank=7 min=4004 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[509, 516, 488, 524, 515, 511, 532, 490, 0, 0, 0, 0, 0, 0, 0, 0, 529, 517, 542, 532, 483, 520, 520, 524, 0, 0, 0, 0, 0, 0, 0, 0, 496, 500, 513, 465, 509, 466, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 513, 521, 525, 518, 492, 535, 507, 525, 0, 0, 0, 0, 0, 0, 0, 0, 515, 513, 517, 510, 494, 537, 538, 533, 0, 0, 0, 0, 0, 0, 0, 0, 516, 507, 531, 514, 522, 507, 516, 496, 0, 0, 0, 0, 0, 0, 0, 0, 498, 543, 497, 498, 487, 489, 512, 512, 33, 26, 13, 0, 0, 0, 0, 0, 495, 499, 497, 499, 478, 529, 469, 502, 16, 9, 6, 5, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4167, 4002, 4136, 4157, 4109, 4108, 4004] avg=4096.0000 max=4167 max_rank=1 min=4002 min_rank=2 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16378 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[578, 518, 513, 528, 525, 525, 525, 519, 0, 0, 0, 0, 0, 0, 0, 0, 521, 495, 484, 482, 540, 518, 509, 540, 0, 0, 0, 0, 0, 0, 0, 0, 490, 512, 516, 501, 504, 506, 479, 519, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 494, 511, 526, 491, 527, 493, 0, 0, 0, 0, 0, 0, 0, 0, 535, 482, 508, 545, 525, 526, 510, 518, 0, 0, 0, 0, 0, 0, 0, 0, 530, 495, 523, 528, 535, 510, 499, 489, 0, 0, 0, 0, 0, 0, 0, 0, 460, 507, 537, 503, 485, 480, 505, 484, 28, 24, 9, 0, 0, 0, 0, 0, 509, 535, 509, 497, 479, 506, 548, 495, 13, 6, 2, 2, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4231, 4089, 4027, 4040, 4149, 4109, 4022, 4101] avg=4096.0000 max=4231 max_rank=0 min=4022 min_rank=6 imbalance_ratio=1.032959 max_load_reduction=28537 max_load_reduction_pct=0.870880 moved_to_echo=16381 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=6 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 3968, 4096, 256, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 3968, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 3968, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 128, 128, 128, 128, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[510, 532, 473, 517, 485, 511, 532, 518, 512, 529, 539, 570, 537, 524, 528, 486, 525, 518, 523, 501, 474, 489, 479, 518, 513, 510, 505, 518, 474, 551, 518, 536, 487, 483, 509, 515, 514, 511, 513, 483, 535, 512, 493, 504, 529, 476, 536, 549, 469, 535, 531, 495, 529, 515, 528, 520, 496, 532, 539, 512, 449, 472, 518, 524] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[538, 544, 510, 491, 521, 505, 496, 515, 515, 508, 536, 536, 526, 480, 525, 518, 501, 514, 521, 505, 475, 518, 508, 506, 543, 507, 494, 534, 486, 522, 512, 529, 518, 512, 518, 532, 501, 510, 523, 490, 492, 538, 505, 532, 522, 514, 502, 516, 512, 510, 515, 526, 499, 499, 481, 510, 522, 483, 492, 468, 574, 508, 476, 529] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[542, 489, 490, 540, 494, 547, 533, 508, 519, 527, 520, 541, 532, 531, 534, 497, 532, 497, 524, 504, 482, 475, 524, 487, 521, 526, 486, 542, 492, 472, 537, 518, 525, 494, 508, 509, 518, 489, 496, 496, 541, 486, 505, 463, 491, 494, 533, 494, 562, 528, 544, 526, 498, 546, 501, 512, 493, 475, 488, 543, 509, 540, 501, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[546, 490, 537, 517, 489, 490, 485, 475, 545, 508, 490, 500, 506, 480, 535, 516, 535, 511, 518, 493, 518, 535, 509, 529, 532, 489, 532, 499, 501, 549, 503, 533, 568, 476, 479, 541, 523, 497, 517, 497, 507, 500, 508, 503, 496, 497, 512, 489, 532, 529, 530, 459, 526, 522, 508, 500, 526, 540, 526, 511, 500, 524, 494, 536] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[477, 518, 499, 522, 517, 521, 483, 514, 511, 501, 529, 530, 546, 542, 518, 509, 528, 524, 470, 531, 554, 489, 523, 462, 519, 519, 506, 508, 512, 515, 537, 475, 508, 516, 496, 504, 488, 488, 564, 542, 563, 497, 536, 516, 524, 525, 512, 531, 496, 518, 473, 524, 506, 537, 499, 513, 542, 454, 504, 503, 491, 491, 502, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[529, 505, 529, 512, 484, 524, 489, 526, 505, 569, 502, 462, 518, 496, 533, 489, 514, 511, 506, 527, 557, 548, 486, 526, 494, 512, 473, 505, 534, 507, 512, 514, 482, 501, 517, 508, 528, 496, 517, 504, 497, 524, 490, 507, 552, 545, 527, 506, 501, 514, 517, 500, 486, 531, 521, 529, 505, 525, 516, 529, 507, 514, 477, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[515, 537, 541, 551, 521, 521, 505, 488, 503, 476, 552, 510, 507, 540, 520, 533, 481, 499, 492, 506, 522, 509, 482, 516, 527, 475, 527, 529, 519, 542, 546, 516, 503, 490, 531, 546, 509, 526, 512, 523, 492, 465, 495, 531, 482, 532, 522, 533, 487, 516, 495, 519, 532, 494, 510, 508, 495, 498, 498, 484, 530, 497, 509, 496] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[521, 530, 513, 461, 509, 509, 489, 469, 517, 554, 494, 526, 501, 496, 496, 493, 504, 549, 503, 501, 500, 499, 530, 485, 557, 513, 503, 513, 546, 539, 523, 521, 551, 492, 523, 526, 499, 549, 529, 513, 489, 505, 559, 543, 514, 505, 480, 490, 509, 493, 523, 505, 509, 517, 529, 506, 515, 502, 508, 482, 496, 511, 544, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[538, 544, 510, 491, 521, 505, 496, 515, 8, 0, 0, 0, 0, 0, 0, 0, 515, 508, 536, 536, 526, 480, 480, 518, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 521, 505, 475, 518, 508, 506, 24, 0, 0, 0, 0, 0, 0, 0, 511, 507, 494, 534, 486, 522, 512, 529, 0, 0, 0, 0, 0, 0, 0, 0, 518, 512, 518, 532, 501, 510, 523, 490, 4, 0, 0, 0, 0, 0, 0, 0, 492, 538, 505, 532, 522, 514, 502, 516, 0, 0, 0, 0, 0, 0, 0, 0, 512, 510, 515, 526, 499, 495, 481, 510, 0, 0, 0, 0, 0, 0, 0, 0, 522, 483, 492, 468, 574, 508, 476, 529, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4128, 4099, 4072, 4095, 4108, 4121, 4048, 4097] avg=4096.0000 max=4128 max_rank=0 min=4048 min_rank=6 imbalance_ratio=1.007812 max_load_reduction=28640 max_load_reduction_pct=0.874023 moved_to_echo=16374 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[477, 518, 499, 522, 517, 521, 483, 514, 7, 0, 0, 0, 0, 0, 0, 0, 511, 501, 529, 530, 546, 542, 474, 509, 0, 0, 0, 0, 0, 0, 0, 0, 528, 524, 470, 531, 554, 489, 523, 462, 22, 0, 0, 0, 0, 0, 0, 0, 490, 519, 506, 508, 512, 515, 537, 475, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 496, 504, 488, 488, 564, 542, 4, 0, 0, 0, 0, 0, 0, 0, 563, 497, 536, 516, 524, 525, 512, 531, 0, 0, 0, 0, 0, 0, 0, 0, 496, 518, 473, 524, 506, 533, 499, 513, 0, 0, 0, 0, 0, 0, 0, 0, 542, 454, 504, 503, 491, 491, 502, 496, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4142, 4103, 4062, 4110, 4204, 4062, 4027] avg=4096.0000 max=4204 max_rank=5 min=4027 min_rank=7 imbalance_ratio=1.026367 max_load_reduction=28564 max_load_reduction_pct=0.871704 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[542, 489, 490, 540, 494, 547, 533, 508, 7, 0, 0, 0, 0, 0, 0, 0, 519, 527, 520, 541, 532, 531, 489, 497, 0, 0, 0, 0, 0, 0, 0, 0, 532, 497, 524, 504, 482, 475, 524, 487, 23, 0, 0, 0, 0, 0, 0, 0, 491, 526, 486, 542, 492, 472, 537, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 494, 508, 509, 518, 489, 496, 496, 4, 0, 0, 0, 0, 0, 0, 0, 541, 486, 505, 463, 491, 494, 533, 494, 0, 0, 0, 0, 0, 0, 0, 0, 562, 528, 544, 526, 498, 542, 501, 512, 0, 0, 0, 0, 0, 0, 0, 0, 493, 475, 488, 543, 509, 540, 501, 497, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4150, 4156, 4048, 4064, 4039, 4007, 4213, 4091] avg=4096.0000 max=4213 max_rank=6 min=4007 min_rank=5 imbalance_ratio=1.028564 max_load_reduction=28555 max_load_reduction_pct=0.871429 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 532, 473, 517, 485, 511, 532, 518, 7, 0, 0, 0, 0, 0, 0, 0, 512, 529, 539, 570, 537, 524, 483, 486, 0, 0, 0, 0, 0, 0, 0, 0, 525, 518, 523, 501, 474, 489, 479, 518, 22, 0, 0, 0, 0, 0, 0, 0, 484, 510, 505, 518, 474, 551, 518, 536, 0, 0, 0, 0, 0, 0, 0, 0, 487, 483, 509, 515, 514, 511, 513, 483, 4, 0, 0, 0, 0, 0, 0, 0, 535, 512, 493, 504, 529, 476, 536, 549, 0, 0, 0, 0, 0, 0, 0, 0, 469, 535, 531, 495, 529, 511, 528, 520, 0, 0, 0, 0, 0, 0, 0, 0, 496, 532, 539, 512, 449, 472, 518, 524, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4180, 4049, 4096, 4019, 4134, 4118, 4087] avg=4096.0000 max=4180 max_rank=1 min=4019 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[546, 490, 537, 517, 489, 490, 485, 475, 7, 0, 0, 0, 0, 0, 0, 0, 545, 508, 490, 500, 506, 480, 490, 516, 0, 0, 0, 0, 0, 0, 0, 0, 535, 511, 518, 493, 518, 535, 509, 529, 23, 0, 0, 0, 0, 0, 0, 0, 502, 489, 532, 499, 501, 549, 503, 533, 0, 0, 0, 0, 0, 0, 0, 0, 568, 476, 479, 541, 523, 497, 517, 497, 4, 0, 0, 0, 0, 0, 0, 0, 507, 500, 508, 503, 496, 497, 512, 489, 0, 0, 0, 0, 0, 0, 0, 0, 532, 529, 530, 459, 526, 518, 508, 500, 0, 0, 0, 0, 0, 0, 0, 0, 526, 540, 526, 511, 500, 524, 494, 536, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4035, 4171, 4108, 4102, 4012, 4102, 4202] avg=4096.0000 max=4202 max_rank=7 min=4012 min_rank=5 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[529, 505, 529, 512, 484, 524, 489, 526, 7, 0, 0, 0, 0, 0, 0, 0, 505, 569, 502, 462, 518, 496, 488, 489, 0, 0, 0, 0, 0, 0, 0, 0, 514, 511, 506, 527, 557, 548, 486, 526, 21, 0, 0, 0, 0, 0, 0, 0, 466, 512, 473, 505, 534, 507, 512, 514, 0, 0, 0, 0, 0, 0, 0, 0, 482, 501, 517, 508, 528, 496, 517, 504, 4, 0, 0, 0, 0, 0, 0, 0, 497, 524, 490, 507, 552, 545, 527, 506, 0, 0, 0, 0, 0, 0, 0, 0, 501, 514, 517, 500, 486, 527, 521, 529, 0, 0, 0, 0, 0, 0, 0, 0, 505, 525, 516, 529, 507, 514, 477, 497, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4105, 4029, 4196, 4023, 4057, 4148, 4095, 4115] avg=4096.0000 max=4196 max_rank=2 min=4023 min_rank=3 imbalance_ratio=1.024414 max_load_reduction=28572 max_load_reduction_pct=0.871948 moved_to_echo=16415 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[515, 537, 541, 551, 521, 521, 505, 488, 12, 6, 0, 0, 0, 0, 0, 0, 503, 476, 552, 510, 507, 540, 471, 533, 0, 0, 0, 0, 0, 0, 0, 0, 481, 499, 492, 506, 522, 509, 482, 516, 27, 0, 0, 0, 0, 0, 0, 0, 483, 475, 527, 529, 519, 542, 546, 516, 0, 0, 0, 0, 0, 0, 0, 0, 503, 490, 531, 546, 509, 526, 512, 523, 7, 0, 0, 0, 0, 0, 0, 0, 492, 465, 495, 531, 482, 532, 522, 533, 5, 0, 0, 0, 0, 0, 0, 0, 487, 516, 495, 519, 532, 476, 510, 508, 0, 0, 0, 0, 0, 0, 0, 0, 495, 498, 498, 484, 530, 497, 509, 496, 49, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4092, 4034, 4137, 4147, 4057, 4043, 4061] avg=4096.0000 max=4197 max_rank=0 min=4034 min_rank=2 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[521, 530, 513, 461, 509, 509, 489, 469, 8, 0, 0, 0, 0, 0, 0, 0, 517, 554, 494, 526, 501, 496, 454, 493, 0, 0, 0, 0, 0, 0, 0, 0, 504, 549, 503, 501, 500, 499, 530, 485, 24, 0, 0, 0, 0, 0, 0, 0, 525, 513, 503, 513, 546, 539, 523, 521, 0, 0, 0, 0, 0, 0, 0, 0, 551, 492, 523, 526, 499, 549, 529, 513, 4, 0, 0, 0, 0, 0, 0, 0, 489, 505, 559, 543, 514, 505, 480, 490, 0, 0, 0, 0, 0, 0, 0, 0, 509, 493, 523, 505, 509, 513, 529, 506, 0, 0, 0, 0, 0, 0, 0, 0, 515, 502, 508, 482, 496, 511, 544, 488, 42, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4009, 4035, 4095, 4183, 4186, 4085, 4087, 4088] avg=4096.0000 max=4186 max_rank=4 min=4009 min_rank=0 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 3968, 4096, 4224, 4096, 4096, 4224, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4224, 4224, 4224, 4096, 3840, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True + [2026-08-26 14:57:44] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 285.4 | throughput per GPU (TFLOP/s/GPU): 432.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.494265E+00 | loss scale: 1.0 | grad norm: 3.439 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=1 tokens_per_expert_current_ep_rank=[498, 527, 503, 531, 530, 498, 513, 492, 527, 515, 515, 504, 486, 535, 518, 483, 489, 514, 557, 497, 519, 514, 532, 512, 500, 534, 511, 500, 524, 520, 511, 521, 492, 503, 518, 507, 523, 527, 518, 518, 520, 528, 532, 504, 479, 501, 523, 493, 539, 529, 498, 541, 485, 472, 513, 510, 489, 496, 536, 520, 512, 501, 513, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=1 tokens_per_expert_current_ep_rank=[510, 480, 532, 516, 496, 505, 501, 516, 501, 497, 534, 555, 499, 502, 516, 542, 529, 496, 481, 540, 496, 505, 513, 478, 540, 486, 512, 551, 499, 492, 513, 501, 530, 525, 526, 504, 539, 559, 561, 515, 550, 511, 477, 490, 536, 526, 556, 517, 462, 492, 483, 545, 469, 496, 521, 498, 518, 491, 526, 485, 481, 528, 519, 498] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=1 tokens_per_expert_current_ep_rank=[504, 542, 529, 521, 526, 511, 489, 556, 505, 501, 522, 499, 500, 502, 514, 511, 525, 480, 592, 489, 477, 508, 517, 471, 527, 501, 516, 523, 530, 519, 489, 525, 510, 516, 531, 523, 452, 470, 530, 470, 500, 508, 518, 524, 504, 568, 488, 490, 530, 524, 522, 513, 532, 509, 536, 523, 541, 501, 508, 531, 498, 492, 475, 510] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=1 tokens_per_expert_current_ep_rank=[548, 534, 498, 506, 525, 543, 533, 510, 492, 522, 554, 540, 490, 550, 551, 480, 519, 482, 506, 472, 512, 525, 486, 532, 517, 539, 500, 480, 559, 533, 524, 538, 499, 505, 505, 513, 508, 491, 513, 509, 505, 494, 475, 453, 533, 497, 520, 517, 531, 513, 462, 547, 540, 527, 505, 510, 516, 552, 517, 484, 505, 503, 472, 447] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=1 tokens_per_expert_current_ep_rank=[489, 539, 507, 511, 517, 462, 510, 503, 532, 564, 504, 522, 509, 499, 508, 512, 524, 506, 445, 492, 549, 498, 504, 534, 502, 520, 495, 515, 477, 540, 507, 537, 532, 504, 502, 486, 514, 503, 532, 474, 517, 556, 558, 511, 522, 495, 533, 516, 517, 458, 493, 511, 515, 532, 505, 529, 498, 517, 537, 494, 543, 529, 518, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=1 tokens_per_expert_current_ep_rank=[509, 512, 520, 541, 523, 469, 523, 517, 513, 522, 514, 482, 521, 518, 536, 528, 538, 543, 523, 507, 510, 527, 516, 518, 547, 513, 490, 526, 554, 474, 510, 486, 500, 516, 515, 510, 573, 500, 544, 493, 512, 477, 509, 511, 549, 511, 502, 536, 490, 536, 522, 485, 459, 505, 496, 490, 480, 542, 484, 492, 488, 511, 506, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=1 tokens_per_expert_current_ep_rank=[520, 504, 507, 563, 568, 534, 506, 531, 504, 555, 493, 469, 498, 521, 486, 532, 527, 537, 531, 541, 523, 491, 499, 509, 502, 500, 487, 501, 514, 504, 505, 501, 536, 502, 489, 491, 497, 501, 520, 498, 524, 507, 554, 487, 542, 515, 488, 497, 478, 488, 467, 514, 500, 524, 499, 535, 503, 488, 533, 527, 495, 524, 557, 525] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=1 tokens_per_expert_current_ep_rank=[484, 518, 488, 530, 510, 550, 517, 537, 515, 445, 544, 509, 512, 493, 519, 495, 466, 497, 537, 513, 487, 533, 481, 520, 552, 535, 482, 516, 559, 518, 520, 541, 527, 524, 482, 531, 500, 512, 531, 515, 512, 553, 510, 507, 506, 512, 501, 507, 528, 494, 555, 487, 508, 511, 498, 502, 524, 515, 518, 502, 501, 503, 482, 487] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=1 rerouted_tokens_per_expert_current_ep_rank=[510, 480, 532, 483, 496, 505, 501, 516, 0, 0, 0, 0, 0, 0, 0, 0, 501, 497, 526, 555, 499, 502, 516, 542, 0, 0, 0, 0, 0, 0, 0, 0, 529, 496, 481, 540, 496, 505, 513, 478, 8, 1, 0, 0, 0, 0, 0, 0, 540, 486, 512, 551, 477, 492, 513, 501, 0, 0, 0, 0, 0, 0, 0, 0, 530, 525, 526, 504, 539, 559, 561, 515, 0, 0, 0, 0, 0, 0, 0, 0, 550, 511, 477, 490, 524, 526, 556, 517, 0, 0, 0, 0, 0, 0, 0, 0, 462, 492, 483, 545, 469, 496, 521, 498, 19, 11, 0, 0, 0, 0, 0, 0, 518, 491, 526, 485, 481, 528, 519, 498, 33, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4023, 4138, 4047, 4072, 4259, 4151, 3996, 4082] avg=4096.0000 max=4259 max_rank=4 min=3996 min_rank=6 imbalance_ratio=1.039795 max_load_reduction=28509 max_load_reduction_pct=0.870026 moved_to_echo=16488 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=1 rerouted_tokens_per_expert_current_ep_rank=[498, 527, 503, 497, 530, 498, 513, 492, 0, 0, 0, 0, 0, 0, 0, 0, 527, 515, 508, 504, 486, 535, 518, 483, 0, 0, 0, 0, 0, 0, 0, 0, 489, 514, 557, 497, 519, 514, 532, 512, 7, 1, 0, 0, 0, 0, 0, 0, 500, 534, 511, 500, 501, 520, 511, 521, 0, 0, 0, 0, 0, 0, 0, 0, 492, 503, 518, 507, 523, 527, 518, 518, 0, 0, 0, 0, 0, 0, 0, 0, 520, 528, 532, 504, 468, 501, 523, 493, 0, 0, 0, 0, 0, 0, 0, 0, 539, 529, 498, 541, 485, 472, 513, 510, 20, 10, 0, 0, 0, 0, 0, 0, 489, 496, 536, 520, 512, 501, 513, 498, 34, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4076, 4142, 4098, 4106, 4069, 4117, 4102] avg=4096.0000 max=4142 max_rank=2 min=4058 min_rank=0 imbalance_ratio=1.011230 max_load_reduction=28626 max_load_reduction_pct=0.873596 moved_to_echo=16394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=1 rerouted_tokens_per_expert_current_ep_rank=[504, 542, 529, 488, 526, 511, 489, 556, 0, 0, 0, 0, 0, 0, 0, 0, 505, 501, 515, 499, 500, 502, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 525, 480, 592, 489, 477, 508, 517, 471, 7, 1, 0, 0, 0, 0, 0, 0, 527, 501, 516, 523, 507, 519, 489, 525, 0, 0, 0, 0, 0, 0, 0, 0, 510, 516, 531, 523, 452, 470, 530, 470, 0, 0, 0, 0, 0, 0, 0, 0, 500, 508, 518, 524, 492, 568, 488, 490, 0, 0, 0, 0, 0, 0, 0, 0, 530, 524, 522, 513, 532, 509, 536, 523, 20, 11, 0, 0, 0, 0, 0, 0, 541, 501, 508, 531, 498, 492, 475, 510, 33, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4145, 4047, 4067, 4107, 4002, 4088, 4220, 4092] avg=4096.0000 max=4220 max_rank=6 min=4002 min_rank=4 imbalance_ratio=1.030273 max_load_reduction=28548 max_load_reduction_pct=0.871216 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=1 rerouted_tokens_per_expert_current_ep_rank=[520, 504, 507, 527, 568, 534, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 504, 555, 486, 469, 498, 521, 486, 532, 0, 0, 0, 0, 0, 0, 0, 0, 527, 537, 531, 541, 523, 491, 499, 509, 7, 1, 0, 0, 0, 0, 0, 0, 502, 500, 487, 501, 491, 504, 505, 501, 0, 0, 0, 0, 0, 0, 0, 0, 536, 502, 489, 491, 497, 501, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 524, 507, 554, 487, 529, 515, 488, 497, 0, 0, 0, 0, 0, 0, 0, 0, 478, 488, 467, 514, 500, 524, 499, 535, 20, 12, 0, 0, 0, 0, 0, 0, 503, 488, 533, 527, 495, 524, 557, 525, 36, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4197, 4051, 4166, 3991, 4034, 4101, 4037, 4191] avg=4096.0000 max=4197 max_rank=0 min=3991 min_rank=3 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16363 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=1 rerouted_tokens_per_expert_current_ep_rank=[548, 534, 498, 474, 525, 543, 533, 510, 0, 0, 0, 0, 0, 0, 0, 0, 492, 522, 546, 540, 490, 550, 551, 480, 0, 0, 0, 0, 0, 0, 0, 0, 519, 482, 506, 472, 512, 525, 486, 532, 8, 1, 0, 0, 0, 0, 0, 0, 517, 539, 500, 480, 534, 533, 524, 538, 0, 0, 0, 0, 0, 0, 0, 0, 499, 505, 505, 513, 508, 491, 513, 509, 0, 0, 0, 0, 0, 0, 0, 0, 505, 494, 475, 453, 521, 497, 520, 517, 0, 0, 0, 0, 0, 0, 0, 0, 531, 513, 462, 547, 540, 527, 505, 510, 22, 11, 0, 0, 0, 0, 0, 0, 516, 552, 517, 484, 505, 503, 472, 447, 32, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4165, 4171, 4043, 4165, 4043, 3982, 4168, 4031] avg=4096.0000 max=4171 max_rank=1 min=3982 min_rank=5 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16224 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=1 rerouted_tokens_per_expert_current_ep_rank=[509, 512, 520, 506, 523, 469, 523, 517, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 507, 482, 521, 518, 536, 528, 0, 0, 0, 0, 0, 0, 0, 0, 538, 543, 523, 507, 510, 527, 516, 518, 7, 1, 0, 0, 0, 0, 0, 0, 547, 513, 490, 526, 530, 474, 510, 486, 0, 0, 0, 0, 0, 0, 0, 0, 500, 516, 515, 510, 573, 500, 544, 493, 0, 0, 0, 0, 0, 0, 0, 0, 512, 477, 509, 511, 536, 511, 502, 536, 0, 0, 0, 0, 0, 0, 0, 0, 490, 536, 522, 485, 459, 505, 496, 490, 21, 12, 0, 0, 0, 0, 0, 0, 480, 542, 484, 492, 488, 511, 506, 494, 35, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4079, 4127, 4190, 4076, 4151, 4094, 4016, 4035] avg=4096.0000 max=4190 max_rank=2 min=4016 min_rank=6 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16296 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=1 rerouted_tokens_per_expert_current_ep_rank=[489, 539, 507, 474, 517, 462, 510, 503, 0, 0, 0, 0, 0, 0, 0, 0, 532, 564, 488, 522, 509, 499, 508, 512, 0, 0, 0, 0, 0, 0, 0, 0, 524, 506, 445, 492, 549, 498, 504, 534, 12, 6, 0, 0, 0, 0, 0, 0, 502, 520, 495, 515, 445, 540, 507, 537, 0, 0, 0, 0, 0, 0, 0, 0, 532, 504, 502, 486, 514, 503, 532, 474, 4, 0, 0, 0, 0, 0, 0, 0, 517, 556, 558, 511, 501, 495, 533, 516, 0, 0, 0, 0, 0, 0, 0, 0, 517, 458, 493, 511, 515, 532, 505, 529, 23, 15, 0, 0, 0, 0, 0, 0, 498, 517, 537, 494, 543, 529, 518, 484, 37, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4001, 4134, 4070, 4061, 4051, 4187, 4098, 4166] avg=4096.0000 max=4187 max_rank=5 min=4001 min_rank=0 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16502 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=1 rerouted_tokens_per_expert_current_ep_rank=[484, 518, 488, 496, 510, 550, 517, 537, 0, 0, 0, 0, 0, 0, 0, 0, 515, 445, 536, 509, 512, 493, 519, 495, 0, 0, 0, 0, 0, 0, 0, 0, 466, 497, 537, 513, 487, 533, 481, 520, 8, 1, 0, 0, 0, 0, 0, 0, 552, 535, 482, 516, 534, 518, 520, 541, 0, 0, 0, 0, 0, 0, 0, 0, 527, 524, 482, 531, 500, 512, 531, 515, 0, 0, 0, 0, 0, 0, 0, 0, 512, 553, 510, 507, 494, 512, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 528, 494, 555, 487, 508, 511, 498, 502, 22, 11, 0, 0, 0, 0, 0, 0, 524, 515, 518, 502, 501, 503, 482, 487, 34, 3, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4100, 4024, 4043, 4198, 4122, 4096, 4116, 4069] avg=4096.0000 max=4198 max_rank=3 min=4024 min_rank=1 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16403 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4096, 4224, 4096, 3968, 4224, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4096, 4096, 3968, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4096, 4352, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[526, 538, 509, 480, 545, 532, 518, 526, 539, 457, 523, 545, 494, 541, 499, 549, 487, 500, 491, 511, 475, 502, 504, 508, 514, 515, 497, 563, 537, 507, 523, 534, 484, 501, 477, 516, 508, 501, 495, 491, 536, 522, 526, 504, 552, 490, 500, 520, 530, 496, 487, 518, 502, 477, 476, 505, 508, 502, 515, 545, 500, 535, 545, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[508, 509, 502, 486, 500, 533, 518, 505, 507, 533, 568, 481, 489, 499, 533, 521, 548, 524, 503, 511, 538, 525, 524, 482, 522, 519, 475, 521, 512, 535, 522, 519, 466, 531, 500, 545, 549, 514, 487, 497, 494, 496, 538, 491, 550, 503, 501, 521, 488, 487, 525, 497, 526, 500, 543, 502, 541, 487, 485, 482, 518, 457, 537, 538] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[492, 509, 477, 526, 511, 524, 529, 515, 503, 538, 539, 525, 524, 465, 521, 459, 497, 519, 497, 535, 512, 498, 522, 487, 527, 522, 505, 490, 519, 502, 515, 499, 513, 492, 491, 513, 555, 526, 534, 510, 537, 528, 518, 520, 520, 503, 496, 499, 503, 511, 517, 504, 519, 512, 519, 518, 521, 540, 475, 521, 479, 515, 529, 527] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[526, 485, 528, 487, 490, 541, 500, 514, 520, 489, 523, 541, 545, 515, 541, 497, 531, 513, 497, 540, 493, 551, 499, 534, 502, 489, 525, 530, 527, 519, 473, 512, 498, 495, 523, 542, 495, 506, 509, 504, 490, 499, 517, 520, 518, 502, 536, 557, 497, 498, 525, 465, 534, 527, 505, 494, 520, 526, 499, 503, 526, 518, 460, 483] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[509, 526, 536, 504, 534, 532, 480, 547, 498, 554, 517, 479, 519, 505, 571, 538, 517, 527, 507, 499, 521, 514, 511, 541, 514, 529, 525, 511, 556, 515, 482, 513, 531, 475, 508, 504, 511, 516, 503, 524, 490, 534, 515, 520, 509, 545, 528, 491, 483, 510, 501, 462, 500, 524, 494, 497, 482, 470, 485, 484, 543, 504, 499, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[518, 518, 497, 523, 527, 558, 522, 502, 525, 485, 507, 524, 556, 505, 526, 471, 507, 511, 489, 501, 526, 558, 513, 494, 532, 513, 486, 501, 493, 521, 499, 531, 536, 525, 515, 500, 538, 516, 527, 513, 497, 474, 474, 520, 503, 537, 516, 503, 508, 504, 508, 514, 491, 514, 507, 527, 516, 513, 501, 500, 502, 502, 511, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[510, 544, 533, 499, 557, 522, 534, 510, 504, 479, 517, 512, 491, 511, 492, 523, 541, 527, 515, 511, 509, 493, 508, 513, 515, 514, 529, 523, 510, 537, 520, 488, 528, 543, 520, 502, 519, 482, 483, 500, 497, 490, 527, 528, 454, 510, 510, 508, 519, 475, 510, 494, 516, 525, 502, 495, 501, 565, 531, 519, 504, 467, 533, 520] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[518, 540, 551, 508, 521, 542, 494, 501, 493, 536, 517, 528, 514, 488, 494, 508, 511, 559, 488, 532, 512, 506, 514, 528, 518, 514, 530, 503, 554, 503, 491, 499, 496, 501, 548, 521, 529, 536, 511, 531, 476, 474, 524, 487, 506, 494, 541, 494, 486, 525, 526, 480, 506, 525, 497, 502, 542, 497, 502, 491, 488, 517, 477, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 538, 509, 480, 545, 491, 518, 526, 0, 0, 0, 0, 0, 0, 0, 0, 539, 457, 506, 545, 494, 541, 499, 549, 0, 0, 0, 0, 0, 0, 0, 0, 487, 490, 491, 511, 475, 502, 504, 508, 0, 0, 0, 0, 0, 0, 0, 0, 514, 515, 497, 563, 516, 507, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 484, 501, 477, 516, 508, 501, 495, 491, 0, 0, 0, 0, 0, 0, 0, 0, 536, 522, 526, 504, 552, 490, 500, 520, 10, 0, 0, 0, 0, 0, 0, 0, 530, 496, 487, 518, 502, 477, 476, 505, 41, 12, 0, 0, 0, 0, 0, 0, 508, 502, 515, 545, 500, 535, 545, 515, 17, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4130, 3968, 4169, 3973, 4160, 4044, 4191] avg=4096.0000 max=4191 max_rank=7 min=3968 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[508, 509, 502, 486, 500, 491, 518, 505, 0, 0, 0, 0, 0, 0, 0, 0, 507, 533, 550, 481, 489, 499, 533, 521, 0, 0, 0, 0, 0, 0, 0, 0, 548, 513, 503, 511, 538, 525, 524, 482, 0, 0, 0, 0, 0, 0, 0, 0, 522, 519, 475, 521, 492, 535, 522, 519, 0, 0, 0, 0, 0, 0, 0, 0, 466, 531, 500, 545, 549, 514, 487, 497, 1, 0, 0, 0, 0, 0, 0, 0, 494, 496, 538, 491, 550, 503, 501, 521, 10, 0, 0, 0, 0, 0, 0, 0, 488, 487, 525, 497, 526, 500, 543, 502, 42, 11, 0, 0, 0, 0, 0, 0, 541, 487, 485, 482, 518, 457, 537, 538, 18, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4019, 4113, 4144, 4105, 4090, 4104, 4121, 4072] avg=4096.0000 max=4144 max_rank=2 min=4019 min_rank=0 imbalance_ratio=1.011719 max_load_reduction=28624 max_load_reduction_pct=0.873535 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[492, 509, 477, 526, 511, 483, 529, 515, 0, 0, 0, 0, 0, 0, 0, 0, 503, 538, 522, 525, 524, 465, 521, 459, 0, 0, 0, 0, 0, 0, 0, 0, 497, 509, 497, 535, 512, 498, 522, 487, 0, 0, 0, 0, 0, 0, 0, 0, 527, 522, 505, 490, 499, 502, 515, 499, 0, 0, 0, 0, 0, 0, 0, 0, 513, 492, 491, 513, 555, 526, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 537, 528, 518, 520, 520, 503, 496, 499, 10, 0, 0, 0, 0, 0, 0, 0, 503, 511, 517, 504, 519, 512, 519, 518, 41, 11, 0, 0, 0, 0, 0, 0, 521, 540, 475, 521, 479, 515, 529, 527, 17, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4042, 4057, 4057, 4059, 4134, 4131, 4155, 4133] avg=4096.0000 max=4155 max_rank=6 min=4042 min_rank=0 imbalance_ratio=1.014404 max_load_reduction=28613 max_load_reduction_pct=0.873199 moved_to_echo=16553 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[509, 526, 536, 504, 534, 491, 480, 547, 0, 0, 0, 0, 0, 0, 0, 0, 498, 554, 500, 479, 519, 505, 571, 538, 0, 0, 0, 0, 0, 0, 0, 0, 517, 516, 507, 499, 521, 514, 511, 541, 0, 0, 0, 0, 0, 0, 0, 0, 514, 529, 525, 511, 534, 515, 482, 513, 0, 0, 0, 0, 0, 0, 0, 0, 531, 475, 508, 504, 511, 516, 503, 524, 1, 0, 0, 0, 0, 0, 0, 0, 490, 534, 515, 520, 509, 545, 528, 491, 10, 0, 0, 0, 0, 0, 0, 0, 483, 510, 501, 462, 500, 524, 494, 497, 41, 12, 0, 0, 0, 0, 0, 0, 482, 470, 485, 484, 543, 504, 499, 495, 17, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4127, 4164, 4126, 4123, 4073, 4142, 4024, 3989] avg=4096.0000 max=4164 max_rank=1 min=3989 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16228 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[510, 544, 533, 499, 557, 481, 534, 510, 0, 0, 0, 0, 0, 0, 0, 0, 504, 479, 500, 512, 491, 511, 492, 523, 0, 0, 0, 0, 0, 0, 0, 0, 541, 516, 515, 511, 509, 493, 508, 513, 0, 0, 0, 0, 0, 0, 0, 0, 515, 514, 529, 523, 490, 537, 520, 488, 0, 0, 0, 0, 0, 0, 0, 0, 528, 543, 520, 502, 519, 482, 483, 500, 1, 0, 0, 0, 0, 0, 0, 0, 497, 490, 527, 528, 454, 510, 510, 508, 10, 0, 0, 0, 0, 0, 0, 0, 519, 475, 510, 494, 516, 525, 502, 495, 41, 11, 0, 0, 0, 0, 0, 0, 501, 565, 531, 519, 504, 467, 533, 520, 17, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4168, 4012, 4106, 4116, 4078, 4034, 4088, 4166] avg=4096.0000 max=4168 max_rank=0 min=4012 min_rank=1 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16366 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 518, 497, 523, 527, 514, 522, 502, 0, 0, 0, 0, 0, 0, 0, 0, 525, 485, 491, 524, 556, 505, 526, 471, 0, 0, 0, 0, 0, 0, 0, 0, 507, 501, 489, 501, 526, 558, 513, 494, 0, 0, 0, 0, 0, 0, 0, 0, 532, 513, 486, 501, 474, 521, 499, 531, 0, 0, 0, 0, 0, 0, 0, 0, 536, 525, 515, 500, 538, 516, 527, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 474, 474, 520, 503, 537, 516, 503, 10, 0, 0, 0, 0, 0, 0, 0, 508, 504, 508, 514, 491, 514, 507, 527, 44, 11, 0, 0, 0, 0, 0, 0, 516, 513, 501, 500, 502, 502, 511, 517, 16, 8, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4083, 4089, 4057, 4170, 4034, 4128, 4086] avg=4096.0000 max=4170 max_rank=4 min=4034 min_rank=5 imbalance_ratio=1.018066 max_load_reduction=28598 max_load_reduction_pct=0.872742 moved_to_echo=16418 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[526, 485, 528, 487, 490, 495, 500, 514, 0, 0, 0, 0, 0, 0, 0, 0, 520, 489, 500, 541, 545, 515, 541, 497, 0, 0, 0, 0, 0, 0, 0, 0, 531, 495, 497, 540, 493, 551, 499, 534, 0, 0, 0, 0, 0, 0, 0, 0, 502, 489, 525, 530, 499, 519, 473, 512, 0, 0, 0, 0, 0, 0, 0, 0, 498, 495, 523, 542, 495, 506, 509, 504, 4, 0, 0, 0, 0, 0, 0, 0, 490, 499, 517, 520, 518, 502, 536, 557, 14, 3, 0, 0, 0, 0, 0, 0, 497, 498, 525, 465, 534, 527, 505, 494, 46, 16, 0, 0, 0, 0, 0, 0, 520, 526, 499, 503, 526, 518, 460, 483, 20, 12, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4025, 4148, 4140, 4049, 4076, 4156, 4107, 4067] avg=4096.0000 max=4156 max_rank=5 min=4025 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16406 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[518, 540, 551, 508, 521, 500, 494, 501, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 500, 528, 514, 488, 494, 508, 0, 0, 0, 0, 0, 0, 0, 0, 511, 547, 488, 532, 512, 506, 514, 528, 0, 0, 0, 0, 0, 0, 0, 0, 518, 514, 530, 503, 532, 503, 491, 499, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 548, 521, 529, 536, 511, 531, 1, 0, 0, 0, 0, 0, 0, 0, 476, 474, 524, 487, 506, 494, 541, 494, 11, 0, 0, 0, 0, 0, 0, 0, 486, 525, 526, 480, 506, 525, 497, 502, 42, 12, 0, 0, 0, 0, 0, 0, 542, 497, 502, 491, 488, 517, 477, 523, 17, 10, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 4061, 4138, 4090, 4174, 4007, 4101, 4064] avg=4096.0000 max=4174 max_rank=4 min=4007 min_rank=5 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16346 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 3968, 4096, 4224, 4096, 4096, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[509, 523, 495, 508, 508, 546, 516, 486, 518, 495, 507, 516, 501, 558, 528, 525, 513, 500, 508, 505, 511, 474, 522, 526, 500, 533, 507, 511, 537, 533, 469, 511, 539, 487, 508, 507, 540, 535, 536, 478, 536, 521, 502, 475, 496, 542, 481, 524, 524, 505, 463, 531, 479, 523, 503, 496, 515, 497, 519, 529, 518, 523, 515, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[564, 480, 508, 517, 470, 498, 503, 481, 477, 548, 492, 536, 504, 512, 490, 485, 537, 520, 546, 541, 516, 520, 479, 513, 551, 531, 525, 520, 522, 513, 509, 536, 485, 553, 540, 536, 486, 504, 520, 522, 517, 530, 520, 492, 513, 552, 520, 530, 489, 500, 515, 459, 532, 470, 507, 511, 466, 504, 530, 487, 503, 499, 511, 521] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[516, 494, 523, 528, 546, 503, 516, 538, 493, 524, 543, 527, 501, 488, 534, 523, 546, 533, 546, 497, 535, 509, 516, 478, 515, 483, 514, 502, 573, 515, 501, 515, 539, 501, 530, 518, 491, 554, 519, 499, 487, 542, 491, 531, 502, 479, 515, 473, 496, 515, 515, 513, 521, 498, 463, 498, 467, 527, 547, 483, 470, 466, 511, 532] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[527, 541, 520, 532, 537, 523, 540, 518, 525, 469, 513, 516, 533, 478, 515, 538, 507, 488, 521, 511, 497, 522, 571, 520, 509, 478, 556, 486, 537, 552, 521, 506, 496, 478, 488, 479, 492, 486, 495, 482, 488, 509, 493, 501, 513, 566, 481, 507, 519, 478, 553, 533, 534, 534, 488, 503, 510, 510, 501, 528, 494, 499, 514, 509] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[482, 519, 551, 502, 522, 521, 542, 502, 520, 500, 494, 510, 485, 508, 501, 524, 501, 508, 515, 547, 527, 472, 491, 451, 523, 505, 490, 566, 564, 530, 494, 534, 489, 485, 571, 497, 506, 481, 517, 500, 498, 499, 561, 538, 487, 512, 511, 486, 519, 529, 507, 514, 511, 500, 521, 535, 527, 499, 496, 531, 543, 520, 474, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[499, 488, 522, 486, 495, 524, 573, 526, 546, 544, 500, 523, 537, 488, 513, 495, 491, 545, 507, 507, 504, 487, 525, 508, 515, 500, 562, 490, 506, 517, 502, 473, 521, 543, 540, 545, 517, 502, 484, 494, 520, 522, 494, 498, 498, 486, 504, 524, 509, 535, 486, 495, 499, 568, 501, 502, 496, 485, 528, 488, 528, 539, 518, 501] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[509, 507, 524, 548, 509, 506, 515, 498, 528, 516, 504, 494, 527, 505, 467, 499, 538, 517, 519, 494, 494, 509, 483, 513, 517, 512, 522, 516, 526, 523, 513, 534, 528, 487, 525, 517, 505, 543, 538, 515, 563, 510, 521, 456, 513, 495, 498, 523, 505, 512, 524, 520, 511, 509, 485, 509, 501, 515, 574, 474, 485, 518, 489, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[496, 472, 515, 544, 529, 525, 481, 539, 504, 554, 560, 511, 494, 582, 473, 538, 491, 532, 529, 517, 504, 523, 503, 525, 491, 496, 531, 535, 505, 537, 484, 505, 495, 456, 526, 463, 500, 536, 504, 503, 519, 485, 504, 531, 499, 492, 506, 475, 518, 524, 501, 524, 491, 518, 506, 513, 500, 534, 529, 533, 524, 497, 523, 514] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[516, 494, 523, 528, 546, 503, 491, 538, 0, 0, 0, 0, 0, 0, 0, 0, 493, 514, 543, 527, 501, 488, 534, 523, 0, 0, 0, 0, 0, 0, 0, 0, 546, 533, 542, 497, 535, 509, 516, 478, 0, 0, 0, 0, 0, 0, 0, 0, 515, 483, 514, 502, 531, 515, 501, 515, 0, 0, 0, 0, 0, 0, 0, 0, 539, 501, 530, 518, 491, 554, 519, 499, 9, 4, 0, 0, 0, 0, 0, 0, 487, 542, 491, 531, 502, 479, 515, 473, 28, 0, 0, 0, 0, 0, 0, 0, 496, 515, 515, 513, 521, 498, 463, 498, 14, 10, 0, 0, 0, 0, 0, 0, 467, 527, 547, 483, 470, 466, 511, 532, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4139, 4123, 4156, 4076, 4164, 4048, 4043, 4019] avg=4096.0000 max=4164 max_rank=4 min=4019 min_rank=7 imbalance_ratio=1.016602 max_load_reduction=28604 max_load_reduction_pct=0.872925 moved_to_echo=16274 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[564, 480, 508, 517, 470, 498, 478, 481, 0, 0, 0, 0, 0, 0, 0, 0, 477, 538, 492, 536, 504, 512, 490, 485, 0, 0, 0, 0, 0, 0, 0, 0, 537, 520, 542, 541, 516, 520, 479, 513, 0, 0, 0, 0, 0, 0, 0, 0, 551, 531, 525, 520, 484, 513, 509, 536, 0, 0, 0, 0, 0, 0, 0, 0, 485, 553, 540, 536, 486, 504, 520, 522, 9, 4, 0, 0, 0, 0, 0, 0, 517, 530, 520, 492, 513, 552, 520, 530, 25, 0, 0, 0, 0, 0, 0, 0, 489, 500, 515, 459, 532, 470, 507, 511, 13, 10, 0, 0, 0, 0, 0, 0, 466, 504, 530, 487, 503, 499, 511, 521, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3996, 4034, 4168, 4169, 4159, 4199, 4006, 4037] avg=4096.0000 max=4199 max_rank=5 min=3996 min_rank=0 imbalance_ratio=1.025146 max_load_reduction=28569 max_load_reduction_pct=0.871857 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 523, 495, 508, 508, 546, 491, 486, 0, 0, 0, 0, 0, 0, 0, 0, 518, 486, 507, 516, 501, 558, 528, 525, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 504, 505, 511, 474, 522, 526, 0, 0, 0, 0, 0, 0, 0, 0, 500, 533, 507, 511, 498, 533, 469, 511, 0, 0, 0, 0, 0, 0, 0, 0, 539, 487, 508, 507, 540, 535, 536, 478, 8, 4, 0, 0, 0, 0, 0, 0, 536, 521, 502, 475, 496, 542, 481, 524, 26, 0, 0, 0, 0, 0, 0, 0, 524, 505, 463, 531, 479, 523, 503, 496, 13, 10, 0, 0, 0, 0, 0, 0, 515, 497, 519, 529, 518, 523, 515, 522, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4066, 4139, 4055, 4062, 4142, 4103, 4047, 4154] avg=4096.0000 max=4154 max_rank=7 min=4047 min_rank=6 imbalance_ratio=1.014160 max_load_reduction=28614 max_load_reduction_pct=0.873230 moved_to_echo=16446 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[509, 507, 524, 548, 509, 506, 490, 498, 0, 0, 0, 0, 0, 0, 0, 0, 528, 506, 504, 494, 527, 505, 467, 499, 0, 0, 0, 0, 0, 0, 0, 0, 538, 517, 515, 494, 494, 509, 483, 513, 0, 0, 0, 0, 0, 0, 0, 0, 517, 512, 522, 516, 488, 523, 513, 534, 0, 0, 0, 0, 0, 0, 0, 0, 528, 487, 525, 517, 505, 543, 538, 515, 9, 4, 0, 0, 0, 0, 0, 0, 563, 510, 521, 456, 513, 495, 498, 523, 25, 0, 0, 0, 0, 0, 0, 0, 505, 512, 524, 520, 511, 509, 485, 509, 13, 10, 0, 0, 0, 0, 0, 0, 501, 515, 574, 474, 485, 518, 489, 514, 15, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4091, 4030, 4063, 4125, 4171, 4104, 4098, 4086] avg=4096.0000 max=4171 max_rank=4 min=4030 min_rank=1 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16459 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[527, 541, 520, 532, 537, 523, 513, 518, 0, 0, 0, 0, 0, 0, 0, 0, 525, 460, 513, 516, 533, 478, 515, 538, 0, 0, 0, 0, 0, 0, 0, 0, 507, 488, 517, 511, 497, 522, 571, 520, 0, 0, 0, 0, 0, 0, 0, 0, 509, 478, 556, 486, 498, 552, 521, 506, 0, 0, 0, 0, 0, 0, 0, 0, 496, 478, 488, 479, 492, 486, 495, 482, 8, 4, 0, 0, 0, 0, 0, 0, 488, 509, 493, 501, 513, 566, 481, 507, 26, 0, 0, 0, 0, 0, 0, 0, 519, 478, 553, 533, 534, 534, 488, 503, 13, 11, 0, 0, 0, 0, 0, 0, 510, 510, 501, 528, 494, 499, 514, 509, 16, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4211, 4078, 4133, 4106, 3908, 4084, 4166, 4082] avg=4096.0000 max=4211 max_rank=0 min=3908 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16240 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[482, 519, 551, 502, 522, 521, 515, 502, 0, 0, 0, 0, 0, 0, 0, 0, 520, 490, 494, 510, 485, 508, 501, 524, 0, 0, 0, 0, 0, 0, 0, 0, 501, 508, 511, 547, 527, 472, 491, 451, 0, 0, 0, 0, 0, 0, 0, 0, 523, 505, 490, 566, 523, 530, 494, 534, 0, 0, 0, 0, 0, 0, 0, 0, 489, 485, 571, 497, 506, 481, 517, 500, 9, 4, 0, 0, 0, 0, 0, 0, 498, 499, 561, 538, 487, 512, 511, 486, 27, 0, 0, 0, 0, 0, 0, 0, 519, 529, 507, 514, 511, 500, 521, 535, 14, 11, 0, 0, 0, 0, 0, 0, 527, 499, 496, 531, 543, 520, 474, 503, 16, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 4032, 4008, 4165, 4059, 4119, 4161, 4110] avg=4096.0000 max=4165 max_rank=3 min=4008 min_rank=2 imbalance_ratio=1.016846 max_load_reduction=28603 max_load_reduction_pct=0.872894 moved_to_echo=16449 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[499, 488, 522, 486, 495, 524, 534, 526, 0, 0, 0, 0, 0, 0, 0, 0, 546, 525, 500, 523, 537, 488, 513, 495, 0, 0, 0, 0, 0, 0, 0, 0, 491, 545, 498, 507, 504, 487, 525, 508, 0, 0, 0, 0, 0, 0, 0, 0, 515, 500, 562, 490, 459, 517, 502, 473, 0, 0, 0, 0, 0, 0, 0, 0, 521, 543, 540, 545, 517, 502, 484, 494, 13, 9, 0, 0, 0, 0, 0, 0, 520, 522, 494, 498, 498, 486, 504, 524, 30, 0, 0, 0, 0, 0, 0, 0, 509, 535, 486, 495, 499, 568, 501, 502, 17, 17, 0, 0, 0, 0, 0, 0, 496, 485, 528, 488, 528, 539, 518, 501, 22, 6, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4074, 4127, 4065, 4018, 4168, 4076, 4129, 4111] avg=4096.0000 max=4168 max_rank=4 min=4018 min_rank=3 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16484 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[496, 472, 515, 544, 529, 525, 457, 539, 0, 0, 0, 0, 0, 0, 0, 0, 504, 543, 560, 511, 494, 582, 473, 538, 0, 0, 0, 0, 0, 0, 0, 0, 491, 532, 525, 517, 504, 523, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 491, 496, 531, 535, 468, 537, 484, 505, 0, 0, 0, 0, 0, 0, 0, 0, 495, 456, 526, 463, 500, 536, 504, 503, 10, 4, 0, 0, 0, 0, 0, 0, 519, 485, 504, 531, 499, 492, 506, 475, 24, 0, 0, 0, 0, 0, 0, 0, 518, 524, 501, 524, 491, 518, 506, 513, 13, 10, 0, 0, 0, 0, 0, 0, 500, 534, 529, 533, 524, 497, 523, 514, 14, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4077, 4205, 4120, 4047, 3997, 4035, 4118, 4169] avg=4096.0000 max=4205 max_rank=1 min=3997 min_rank=4 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16319 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 3968, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4352, 4096, 4096, 4224, 4224, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[544, 536, 522, 522, 475, 504, 507, 486, 537, 501, 497, 528, 521, 485, 473, 530, 492, 493, 504, 509, 535, 532, 517, 539, 489, 547, 505, 495, 541, 501, 511, 524, 519, 537, 531, 523, 539, 554, 498, 506, 517, 501, 519, 526, 480, 500, 507, 530, 552, 523, 501, 503, 477, 526, 503, 507, 476, 486, 509, 536, 512, 515, 482, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[519, 533, 461, 505, 519, 526, 508, 527, 527, 516, 522, 553, 506, 504, 507, 520, 517, 513, 486, 507, 512, 536, 502, 507, 484, 509, 512, 507, 512, 500, 517, 482, 516, 515, 511, 537, 519, 516, 496, 542, 489, 458, 518, 535, 509, 459, 525, 546, 501, 483, 540, 512, 518, 527, 511, 496, 540, 491, 534, 535, 514, 456, 502, 561] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[498, 502, 486, 517, 508, 521, 548, 530, 527, 500, 502, 547, 484, 543, 515, 495, 510, 530, 492, 513, 501, 490, 522, 516, 481, 515, 523, 536, 502, 539, 484, 524, 506, 521, 529, 517, 519, 536, 500, 523, 498, 484, 514, 519, 508, 506, 502, 519, 508, 507, 505, 520, 521, 532, 525, 506, 519, 527, 507, 483, 479, 526, 513, 488] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[521, 505, 522, 490, 512, 540, 526, 489, 513, 489, 522, 505, 512, 525, 498, 525, 513, 501, 501, 494, 474, 481, 547, 514, 547, 509, 493, 488, 498, 535, 518, 497, 502, 518, 496, 506, 513, 514, 511, 518, 476, 543, 486, 493, 468, 512, 515, 505, 530, 534, 500, 533, 508, 516, 535, 517, 507, 547, 515, 532, 526, 539, 512, 537] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[540, 530, 530, 486, 535, 522, 512, 518, 520, 485, 489, 495, 512, 475, 518, 490, 519, 449, 517, 523, 504, 535, 539, 495, 501, 498, 489, 532, 535, 510, 528, 505, 546, 508, 527, 508, 508, 513, 505, 513, 482, 514, 517, 504, 487, 496, 511, 515, 523, 468, 502, 520, 507, 541, 511, 505, 510, 530, 539, 535, 515, 523, 519, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[505, 520, 512, 529, 489, 559, 544, 507, 516, 487, 520, 495, 564, 538, 483, 498, 501, 520, 503, 514, 527, 546, 482, 534, 494, 480, 494, 520, 479, 522, 536, 488, 483, 524, 475, 512, 508, 513, 519, 543, 515, 498, 520, 490, 517, 496, 509, 527, 513, 495, 513, 577, 496, 492, 517, 501, 535, 513, 537, 483, 538, 500, 490, 513] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[533, 520, 516, 515, 529, 521, 495, 506, 497, 486, 515, 504, 515, 524, 529, 504, 502, 507, 467, 540, 530, 531, 477, 529, 529, 521, 489, 523, 500, 506, 503, 506, 501, 518, 484, 524, 527, 492, 497, 477, 523, 502, 514, 499, 508, 509, 511, 540, 519, 511, 530, 525, 521, 526, 538, 528, 498, 519, 502, 515, 467, 524, 506, 544] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[551, 512, 515, 496, 518, 520, 596, 513, 482, 522, 488, 532, 515, 499, 498, 553, 524, 530, 526, 510, 506, 492, 537, 495, 544, 465, 539, 523, 479, 542, 478, 511, 484, 553, 501, 557, 506, 513, 479, 475, 515, 499, 493, 510, 503, 492, 513, 510, 537, 529, 486, 510, 485, 520, 495, 504, 497, 531, 521, 521, 501, 501, 522, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[519, 533, 461, 505, 519, 526, 468, 527, 0, 0, 0, 0, 0, 0, 0, 0, 527, 516, 522, 553, 506, 504, 507, 520, 8, 3, 0, 0, 0, 0, 0, 0, 517, 513, 486, 507, 512, 536, 502, 507, 7, 0, 0, 0, 0, 0, 0, 0, 484, 509, 512, 507, 512, 500, 517, 482, 15, 5, 0, 0, 0, 0, 0, 0, 516, 502, 511, 537, 519, 516, 496, 542, 0, 0, 0, 0, 0, 0, 0, 0, 489, 458, 518, 535, 509, 459, 525, 546, 40, 3, 0, 0, 0, 0, 0, 0, 501, 483, 540, 494, 518, 527, 511, 496, 0, 0, 0, 0, 0, 0, 0, 0, 540, 491, 524, 535, 514, 456, 502, 561, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4058, 4166, 4087, 4043, 4139, 4082, 4070, 4123] avg=4096.0000 max=4166 max_rank=1 min=4043 min_rank=3 imbalance_ratio=1.017090 max_load_reduction=28602 max_load_reduction_pct=0.872864 moved_to_echo=16414 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[544, 536, 522, 522, 475, 504, 467, 486, 0, 0, 0, 0, 0, 0, 0, 0, 537, 501, 497, 528, 521, 485, 473, 530, 8, 3, 0, 0, 0, 0, 0, 0, 492, 493, 504, 509, 535, 532, 517, 539, 6, 0, 0, 0, 0, 0, 0, 0, 489, 547, 505, 495, 541, 501, 511, 524, 15, 6, 0, 0, 0, 0, 0, 0, 519, 523, 531, 523, 539, 554, 498, 506, 0, 0, 0, 0, 0, 0, 0, 0, 517, 501, 519, 526, 480, 500, 507, 530, 40, 3, 0, 0, 0, 0, 0, 0, 552, 523, 501, 485, 477, 526, 503, 507, 0, 0, 0, 0, 0, 0, 0, 0, 476, 486, 500, 536, 512, 515, 482, 471, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4083, 4127, 4134, 4193, 4123, 4074, 3978] avg=4096.0000 max=4193 max_rank=4 min=3978 min_rank=7 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16368 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[498, 502, 486, 517, 508, 521, 505, 530, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 502, 547, 484, 543, 515, 495, 8, 3, 0, 0, 0, 0, 0, 0, 510, 530, 492, 513, 501, 490, 522, 516, 6, 0, 0, 0, 0, 0, 0, 0, 481, 515, 523, 536, 502, 539, 484, 524, 15, 5, 0, 0, 0, 0, 0, 0, 506, 508, 529, 517, 519, 536, 500, 523, 0, 0, 0, 0, 0, 0, 0, 0, 498, 484, 514, 519, 508, 506, 502, 519, 43, 3, 0, 0, 0, 0, 0, 0, 508, 507, 505, 502, 521, 532, 525, 506, 0, 0, 0, 0, 0, 0, 0, 0, 519, 527, 498, 483, 479, 526, 513, 488, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4067, 4124, 4080, 4124, 4138, 4096, 4106, 4033] avg=4096.0000 max=4138 max_rank=4 min=4033 min_rank=7 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16373 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[521, 505, 522, 490, 512, 540, 485, 489, 0, 0, 0, 0, 0, 0, 0, 0, 513, 489, 522, 505, 512, 525, 498, 525, 8, 3, 0, 0, 0, 0, 0, 0, 513, 501, 501, 494, 474, 481, 547, 514, 7, 0, 0, 0, 0, 0, 0, 0, 547, 509, 493, 488, 498, 535, 518, 497, 16, 5, 0, 0, 0, 0, 0, 0, 502, 505, 496, 506, 513, 514, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 476, 543, 486, 493, 468, 512, 515, 505, 41, 3, 0, 0, 0, 0, 0, 0, 530, 534, 500, 514, 508, 516, 535, 517, 0, 0, 0, 0, 0, 0, 0, 0, 507, 547, 505, 532, 526, 539, 512, 537, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4100, 4032, 4106, 4065, 4042, 4154, 4205] avg=4096.0000 max=4205 max_rank=7 min=4032 min_rank=2 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[540, 530, 530, 486, 535, 522, 472, 518, 0, 0, 0, 0, 0, 0, 0, 0, 520, 485, 489, 495, 512, 475, 518, 490, 7, 3, 0, 0, 0, 0, 0, 0, 519, 449, 517, 523, 504, 535, 539, 495, 7, 0, 0, 0, 0, 0, 0, 0, 501, 498, 489, 532, 535, 510, 528, 505, 15, 5, 0, 0, 0, 0, 0, 0, 546, 496, 527, 508, 508, 513, 505, 513, 0, 0, 0, 0, 0, 0, 0, 0, 482, 514, 517, 504, 487, 496, 511, 515, 40, 3, 0, 0, 0, 0, 0, 0, 523, 468, 502, 502, 507, 541, 511, 505, 0, 0, 0, 0, 0, 0, 0, 0, 510, 530, 529, 535, 515, 523, 519, 530, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4133, 3994, 4088, 4118, 4116, 4069, 4059, 4191] avg=4096.0000 max=4191 max_rank=7 min=3994 min_rank=1 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[533, 520, 516, 515, 529, 521, 456, 506, 0, 0, 0, 0, 0, 0, 0, 0, 497, 486, 515, 504, 515, 524, 529, 504, 8, 3, 0, 0, 0, 0, 0, 0, 502, 507, 467, 540, 530, 531, 477, 529, 6, 0, 0, 0, 0, 0, 0, 0, 529, 521, 489, 523, 500, 506, 503, 506, 15, 5, 0, 0, 0, 0, 0, 0, 501, 505, 484, 524, 527, 492, 497, 477, 0, 0, 0, 0, 0, 0, 0, 0, 523, 502, 514, 499, 508, 509, 511, 540, 39, 3, 0, 0, 0, 0, 0, 0, 519, 511, 530, 507, 521, 526, 538, 528, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 493, 515, 467, 524, 506, 544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4085, 4089, 4097, 4007, 4148, 4180, 4066] avg=4096.0000 max=4180 max_rank=6 min=4007 min_rank=4 imbalance_ratio=1.020508 max_load_reduction=28588 max_load_reduction_pct=0.872437 moved_to_echo=16401 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[505, 520, 512, 529, 489, 559, 499, 507, 0, 0, 0, 0, 0, 0, 0, 0, 516, 487, 520, 495, 564, 538, 483, 498, 11, 4, 0, 0, 0, 0, 0, 0, 501, 520, 503, 514, 527, 546, 482, 534, 11, 0, 0, 0, 0, 0, 0, 0, 494, 480, 494, 520, 479, 522, 536, 488, 21, 10, 0, 0, 0, 0, 0, 0, 483, 503, 475, 512, 508, 513, 519, 543, 0, 0, 0, 0, 0, 0, 0, 0, 515, 498, 520, 490, 517, 496, 509, 527, 45, 6, 0, 0, 0, 0, 0, 0, 513, 495, 513, 550, 496, 492, 517, 501, 0, 0, 0, 0, 0, 0, 0, 0, 535, 513, 522, 483, 538, 500, 490, 513, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4116, 4138, 4044, 4056, 4123, 4077, 4094] avg=4096.0000 max=4138 max_rank=2 min=4044 min_rank=3 imbalance_ratio=1.010254 max_load_reduction=28630 max_load_reduction_pct=0.873718 moved_to_echo=16350 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[551, 512, 515, 496, 518, 520, 549, 513, 0, 0, 0, 0, 0, 0, 0, 0, 482, 522, 488, 532, 515, 499, 498, 553, 8, 3, 0, 0, 0, 0, 0, 0, 524, 530, 526, 510, 506, 492, 537, 495, 7, 0, 0, 0, 0, 0, 0, 0, 544, 465, 539, 523, 479, 542, 478, 511, 15, 6, 0, 0, 0, 0, 0, 0, 484, 539, 501, 557, 506, 513, 479, 475, 0, 0, 0, 0, 0, 0, 0, 0, 515, 499, 493, 510, 503, 492, 513, 510, 47, 3, 0, 0, 0, 0, 0, 0, 537, 529, 486, 492, 485, 520, 495, 504, 0, 0, 0, 0, 0, 0, 0, 0, 497, 531, 511, 521, 501, 501, 522, 494, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4174, 4100, 4127, 4102, 4054, 4085, 4048, 4078] avg=4096.0000 max=4174 max_rank=0 min=4048 min_rank=6 imbalance_ratio=1.019043 max_load_reduction=28594 max_load_reduction_pct=0.872620 moved_to_echo=16265 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33024 observed_loss=-256 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 3968, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4096, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33024 slack_slots=32512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[557, 545, 474, 512, 567, 522, 495, 506, 531, 508, 512, 482, 511, 544, 496, 518, 484, 532, 528, 481, 516, 535, 500, 489, 556, 503, 527, 516, 525, 501, 509, 484, 510, 549, 533, 541, 513, 485, 526, 513, 513, 510, 570, 515, 495, 469, 489, 520, 495, 551, 482, 497, 519, 454, 481, 524, 500, 523, 468, 529, 488, 499, 546, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[497, 486, 506, 528, 528, 518, 515, 508, 527, 509, 503, 525, 466, 527, 529, 490, 499, 521, 519, 515, 531, 544, 491, 509, 514, 490, 516, 502, 538, 535, 544, 494, 497, 480, 518, 511, 494, 515, 522, 522, 495, 508, 511, 511, 531, 466, 577, 503, 518, 466, 475, 535, 500, 511, 524, 522, 532, 497, 531, 568, 482, 498, 506, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[504, 483, 505, 497, 522, 497, 511, 518, 530, 497, 556, 551, 504, 485, 548, 511, 533, 506, 510, 519, 481, 499, 548, 542, 559, 495, 484, 527, 542, 500, 495, 507, 564, 507, 507, 496, 508, 483, 508, 511, 513, 522, 465, 529, 498, 469, 520, 503, 518, 533, 550, 536, 514, 499, 496, 490, 500, 495, 533, 492, 512, 511, 531, 489] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[547, 535, 517, 492, 547, 489, 524, 551, 519, 499, 535, 488, 545, 510, 506, 502, 480, 500, 486, 502, 504, 509, 467, 502, 496, 501, 532, 561, 495, 496, 524, 517, 517, 536, 480, 523, 538, 543, 504, 491, 519, 504, 487, 519, 493, 484, 547, 513, 499, 533, 482, 502, 494, 505, 557, 500, 539, 496, 515, 492, 521, 500, 545, 512] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[475, 507, 525, 540, 495, 520, 543, 548, 521, 487, 527, 505, 554, 520, 520, 523, 525, 515, 487, 502, 528, 522, 524, 546, 504, 532, 477, 513, 522, 514, 508, 507, 495, 527, 488, 487, 529, 506, 537, 496, 493, 497, 538, 493, 530, 486, 543, 518, 475, 522, 486, 487, 461, 528, 530, 495, 469, 522, 518, 509, 521, 498, 554, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[500, 496, 542, 528, 463, 510, 537, 515, 508, 500, 495, 499, 511, 508, 531, 515, 543, 530, 500, 519, 479, 472, 525, 492, 517, 524, 504, 505, 479, 535, 500, 494, 516, 568, 501, 527, 537, 554, 475, 540, 496, 527, 477, 546, 532, 540, 566, 523, 516, 507, 535, 448, 474, 526, 503, 533, 459, 518, 505, 504, 467, 537, 531, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[499, 570, 482, 536, 527, 481, 550, 511, 469, 500, 519, 513, 539, 517, 479, 525, 510, 522, 511, 531, 512, 506, 534, 515, 513, 509, 515, 475, 498, 496, 544, 521, 522, 488, 530, 488, 480, 518, 500, 541, 503, 532, 492, 509, 495, 517, 514, 497, 533, 532, 493, 516, 506, 501, 508, 501, 515, 485, 545, 528, 521, 546, 484, 499] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[482, 519, 507, 490, 541, 549, 494, 524, 532, 506, 512, 537, 503, 497, 498, 552, 534, 523, 500, 534, 520, 519, 514, 496, 487, 569, 525, 522, 500, 535, 522, 482, 532, 517, 520, 502, 472, 491, 489, 516, 484, 543, 508, 486, 514, 527, 499, 488, 505, 518, 489, 506, 522, 478, 490, 545, 497, 511, 505, 513, 531, 514, 524, 507] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[547, 535, 517, 492, 516, 489, 524, 551, 0, 0, 0, 0, 0, 0, 0, 0, 519, 499, 521, 488, 545, 510, 506, 502, 0, 0, 0, 0, 0, 0, 0, 0, 480, 500, 486, 502, 504, 509, 467, 502, 0, 0, 0, 0, 0, 0, 0, 0, 496, 501, 532, 561, 495, 496, 516, 517, 0, 0, 0, 0, 0, 0, 0, 0, 517, 528, 480, 523, 538, 543, 504, 491, 0, 0, 0, 0, 0, 0, 0, 0, 519, 504, 487, 519, 493, 484, 547, 513, 8, 2, 0, 0, 0, 0, 0, 0, 499, 533, 482, 502, 494, 505, 557, 500, 31, 12, 0, 0, 0, 0, 0, 0, 539, 496, 515, 492, 521, 500, 545, 512, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4171, 4090, 3950, 4114, 4124, 4076, 4115, 4128] avg=4096.0000 max=4171 max_rank=0 min=3950 min_rank=2 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[557, 545, 474, 512, 535, 522, 495, 506, 0, 0, 0, 0, 0, 0, 0, 0, 531, 508, 499, 482, 511, 544, 496, 518, 0, 0, 0, 0, 0, 0, 0, 0, 484, 532, 528, 481, 516, 535, 500, 489, 0, 0, 0, 0, 0, 0, 0, 0, 556, 503, 527, 516, 525, 501, 502, 484, 0, 0, 0, 0, 0, 0, 0, 0, 510, 541, 533, 541, 513, 485, 526, 513, 0, 0, 0, 0, 0, 0, 0, 0, 513, 510, 570, 515, 495, 469, 489, 520, 8, 2, 0, 0, 0, 0, 0, 0, 495, 551, 482, 497, 519, 454, 481, 524, 32, 11, 0, 0, 0, 0, 0, 0, 500, 523, 468, 529, 488, 499, 546, 495, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4146, 4089, 4065, 4114, 4162, 4091, 4046, 4055] avg=4096.0000 max=4162 max_rank=4 min=4046 min_rank=6 imbalance_ratio=1.016113 max_load_reduction=28606 max_load_reduction_pct=0.872986 moved_to_echo=16354 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[504, 483, 505, 497, 492, 497, 511, 518, 0, 0, 0, 0, 0, 0, 0, 0, 530, 497, 542, 551, 504, 485, 548, 511, 0, 0, 0, 0, 0, 0, 0, 0, 533, 506, 510, 519, 481, 499, 548, 542, 0, 0, 0, 0, 0, 0, 0, 0, 559, 495, 484, 527, 542, 500, 488, 507, 0, 0, 0, 0, 0, 0, 0, 0, 564, 500, 507, 496, 508, 483, 508, 511, 0, 0, 0, 0, 0, 0, 0, 0, 513, 522, 465, 529, 498, 469, 520, 503, 7, 2, 0, 0, 0, 0, 0, 0, 518, 533, 550, 536, 514, 499, 496, 490, 30, 12, 0, 0, 0, 0, 0, 0, 500, 495, 533, 492, 512, 511, 531, 489, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4007, 4168, 4138, 4102, 4077, 4028, 4178, 4070] avg=4096.0000 max=4178 max_rank=6 min=4007 min_rank=0 imbalance_ratio=1.020020 max_load_reduction=28590 max_load_reduction_pct=0.872498 moved_to_echo=16353 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[475, 507, 525, 540, 467, 520, 543, 548, 0, 0, 0, 0, 0, 0, 0, 0, 521, 487, 513, 505, 554, 520, 520, 523, 0, 0, 0, 0, 0, 0, 0, 0, 525, 515, 487, 502, 528, 522, 524, 546, 0, 0, 0, 0, 0, 0, 0, 0, 504, 532, 477, 513, 522, 514, 501, 507, 0, 0, 0, 0, 0, 0, 0, 0, 495, 519, 488, 487, 529, 506, 537, 496, 0, 0, 0, 0, 0, 0, 0, 0, 493, 497, 538, 493, 530, 486, 543, 518, 8, 2, 0, 0, 0, 0, 0, 0, 475, 522, 486, 487, 461, 528, 530, 495, 28, 12, 0, 0, 0, 0, 0, 0, 469, 522, 518, 509, 521, 498, 554, 494, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4125, 4143, 4149, 4070, 4057, 4108, 4024, 4092] avg=4096.0000 max=4149 max_rank=2 min=4024 min_rank=6 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16281 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[497, 486, 506, 528, 498, 518, 515, 508, 0, 0, 0, 0, 0, 0, 0, 0, 527, 509, 490, 525, 466, 527, 529, 490, 0, 0, 0, 0, 0, 0, 0, 0, 499, 521, 519, 515, 531, 544, 491, 509, 0, 0, 0, 0, 0, 0, 0, 0, 514, 490, 516, 502, 538, 535, 536, 494, 0, 0, 0, 0, 0, 0, 0, 0, 497, 473, 518, 511, 494, 515, 522, 522, 0, 0, 0, 0, 0, 0, 0, 0, 495, 508, 511, 511, 531, 466, 577, 503, 7, 2, 0, 0, 0, 0, 0, 0, 518, 466, 475, 535, 500, 511, 524, 522, 30, 11, 0, 0, 0, 0, 0, 0, 532, 497, 531, 568, 482, 498, 506, 518, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4056, 4063, 4129, 4125, 4052, 4111, 4092, 4140] avg=4096.0000 max=4140 max_rank=7 min=4052 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16395 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[500, 496, 542, 528, 437, 510, 537, 515, 0, 0, 0, 0, 0, 0, 0, 0, 508, 500, 482, 499, 511, 508, 531, 515, 0, 0, 0, 0, 0, 0, 0, 0, 543, 530, 500, 519, 479, 472, 525, 492, 0, 0, 0, 0, 0, 0, 0, 0, 517, 524, 504, 505, 479, 535, 493, 494, 0, 0, 0, 0, 0, 0, 0, 0, 516, 560, 501, 527, 537, 554, 475, 540, 0, 0, 0, 0, 0, 0, 0, 0, 496, 527, 477, 546, 532, 540, 566, 523, 8, 2, 0, 0, 0, 0, 0, 0, 516, 507, 535, 448, 474, 526, 503, 533, 26, 11, 0, 0, 0, 0, 0, 0, 459, 518, 505, 504, 467, 537, 531, 504, 7, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4054, 4060, 4051, 4210, 4217, 4079, 4032] avg=4096.0000 max=4217 max_rank=5 min=4032 min_rank=7 imbalance_ratio=1.029541 max_load_reduction=28551 max_load_reduction_pct=0.871307 moved_to_echo=16538 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[499, 570, 482, 536, 494, 481, 550, 511, 0, 0, 0, 0, 0, 0, 0, 0, 469, 500, 495, 513, 539, 517, 479, 525, 0, 0, 0, 0, 0, 0, 0, 0, 510, 518, 511, 531, 512, 506, 534, 515, 0, 0, 0, 0, 0, 0, 0, 0, 513, 509, 515, 475, 498, 496, 531, 521, 0, 0, 0, 0, 0, 0, 0, 0, 522, 476, 530, 488, 480, 518, 500, 541, 0, 0, 0, 0, 0, 0, 0, 0, 503, 532, 492, 509, 495, 517, 514, 497, 11, 8, 0, 0, 0, 0, 0, 0, 533, 532, 493, 516, 506, 501, 508, 501, 33, 16, 0, 0, 0, 0, 0, 0, 515, 485, 545, 528, 521, 546, 484, 499, 13, 4, 1, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4123, 4037, 4137, 4058, 4055, 4078, 4139, 4141] avg=4096.0000 max=4141 max_rank=7 min=4037 min_rank=1 imbalance_ratio=1.010986 max_load_reduction=28627 max_load_reduction_pct=0.873627 moved_to_echo=16413 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[482, 519, 507, 490, 510, 549, 494, 524, 0, 0, 0, 0, 0, 0, 0, 0, 532, 506, 499, 537, 503, 497, 498, 552, 0, 0, 0, 0, 0, 0, 0, 0, 534, 523, 500, 534, 520, 519, 514, 496, 0, 0, 0, 0, 0, 0, 0, 0, 487, 569, 525, 522, 500, 535, 514, 482, 0, 0, 0, 0, 0, 0, 0, 0, 532, 509, 520, 502, 472, 491, 489, 516, 0, 0, 0, 0, 0, 0, 0, 0, 484, 543, 508, 486, 514, 527, 499, 488, 8, 2, 0, 0, 0, 0, 0, 0, 505, 518, 489, 506, 522, 478, 490, 545, 31, 11, 0, 0, 0, 0, 0, 0, 497, 511, 505, 513, 531, 514, 524, 507, 8, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4075, 4124, 4140, 4134, 4031, 4059, 4095, 4110] avg=4096.0000 max=4140 max_rank=2 min=4031 min_rank=4 imbalance_ratio=1.010742 max_load_reduction=28628 max_load_reduction_pct=0.873657 moved_to_echo=16295 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 3968, 4352, 4096, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4096, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4096, 4096, 4224, 4224, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4224, 3968, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[517, 551, 482, 530, 490, 502, 522, 515, 525, 505, 543, 507, 516, 510, 527, 489, 555, 534, 512, 504, 505, 547, 508, 538, 520, 488, 515, 516, 513, 495, 487, 508, 508, 504, 491, 524, 494, 503, 526, 506, 508, 518, 496, 472, 521, 516, 521, 504, 504, 499, 505, 560, 537, 527, 488, 519, 545, 496, 488, 531, 478, 496, 504, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[552, 479, 552, 526, 492, 506, 502, 482, 524, 492, 516, 478, 525, 507, 505, 526, 513, 500, 507, 528, 525, 513, 515, 544, 500, 519, 508, 503, 515, 519, 526, 484, 517, 520, 497, 483, 538, 548, 518, 481, 526, 513, 488, 488, 504, 541, 517, 513, 562, 553, 548, 532, 525, 496, 507, 490, 503, 482, 505, 508, 459, 525, 505, 493] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[501, 489, 537, 482, 489, 503, 548, 494, 512, 478, 494, 526, 554, 517, 544, 511, 483, 515, 500, 548, 506, 483, 499, 487, 537, 530, 517, 527, 497, 501, 516, 490, 521, 489, 526, 504, 524, 514, 518, 499, 528, 489, 551, 513, 482, 531, 511, 487, 500, 524, 508, 508, 506, 531, 525, 520, 540, 518, 533, 541, 496, 524, 520, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[525, 488, 488, 525, 555, 457, 533, 505, 533, 529, 507, 501, 501, 490, 514, 494, 525, 497, 501, 544, 537, 571, 510, 504, 519, 537, 505, 533, 469, 500, 536, 512, 502, 513, 504, 490, 518, 527, 529, 534, 520, 518, 514, 497, 485, 499, 524, 507, 472, 516, 536, 539, 539, 505, 552, 487, 501, 512, 498, 536, 468, 501, 483, 497] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[511, 527, 492, 488, 553, 516, 527, 483, 498, 526, 509, 512, 514, 503, 530, 530, 511, 551, 500, 597, 528, 528, 498, 530, 531, 482, 497, 494, 536, 463, 507, 527, 555, 484, 525, 473, 511, 515, 486, 505, 491, 495, 494, 476, 525, 541, 491, 531, 516, 537, 506, 478, 544, 510, 487, 505, 514, 503, 535, 511, 508, 516, 511, 490] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[525, 489, 529, 524, 491, 494, 513, 502, 503, 497, 528, 560, 525, 507, 548, 552, 490, 527, 527, 536, 507, 480, 484, 496, 482, 465, 542, 558, 519, 474, 521, 506, 530, 549, 515, 520, 546, 535, 459, 543, 504, 506, 525, 494, 490, 479, 471, 480, 532, 536, 515, 515, 539, 520, 516, 521, 512, 512, 512, 476, 503, 532, 462, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[517, 513, 475, 494, 493, 500, 493, 492, 521, 559, 488, 513, 521, 531, 560, 528, 574, 526, 527, 511, 496, 478, 508, 555, 525, 539, 468, 522, 520, 478, 469, 507, 490, 523, 524, 521, 480, 518, 464, 516, 521, 542, 532, 502, 523, 507, 499, 514, 480, 534, 535, 521, 521, 513, 523, 482, 513, 501, 508, 540, 499, 493, 517, 511] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[513, 549, 499, 495, 525, 549, 478, 532, 532, 524, 527, 482, 498, 536, 502, 503, 545, 468, 513, 519, 514, 568, 515, 535, 477, 492, 502, 499, 505, 500, 521, 522, 510, 533, 511, 507, 510, 485, 522, 529, 455, 528, 510, 530, 521, 522, 495, 482, 512, 506, 497, 507, 512, 575, 503, 525, 514, 515, 503, 479, 485, 504, 508, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 488, 488, 525, 555, 457, 533, 505, 20, 0, 0, 0, 0, 0, 0, 0, 533, 529, 507, 501, 501, 490, 479, 494, 0, 0, 0, 0, 0, 0, 0, 0, 525, 497, 501, 490, 537, 571, 510, 504, 0, 0, 0, 0, 0, 0, 0, 0, 519, 537, 505, 533, 469, 500, 536, 512, 19, 15, 0, 0, 0, 0, 0, 0, 502, 513, 504, 490, 518, 527, 529, 534, 0, 0, 0, 0, 0, 0, 0, 0, 520, 518, 514, 497, 485, 499, 524, 507, 28, 8, 0, 0, 0, 0, 0, 0, 472, 516, 536, 539, 492, 505, 552, 487, 0, 0, 0, 0, 0, 0, 0, 0, 501, 512, 498, 536, 468, 501, 483, 497, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4096, 4034, 4135, 4145, 4117, 4100, 4099, 4042] avg=4096.0000 max=4145 max_rank=3 min=4034 min_rank=1 imbalance_ratio=1.011963 max_load_reduction=28623 max_load_reduction_pct=0.873505 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[501, 489, 537, 482, 489, 503, 548, 494, 21, 0, 0, 0, 0, 0, 0, 0, 512, 478, 494, 526, 554, 517, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 483, 515, 500, 493, 506, 483, 499, 487, 0, 0, 0, 0, 0, 0, 0, 0, 537, 530, 517, 527, 497, 501, 516, 490, 17, 16, 0, 0, 0, 0, 0, 0, 521, 489, 526, 504, 524, 514, 518, 499, 0, 0, 0, 0, 0, 0, 0, 0, 528, 489, 551, 513, 482, 531, 511, 487, 27, 8, 0, 0, 0, 0, 0, 0, 500, 524, 508, 508, 462, 531, 525, 520, 0, 0, 0, 0, 0, 0, 0, 0, 540, 518, 533, 541, 496, 524, 520, 472, 47, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4064, 4099, 3966, 4148, 4095, 4127, 4078, 4191] avg=4096.0000 max=4191 max_rank=7 min=3966 min_rank=2 imbalance_ratio=1.023193 max_load_reduction=28577 max_load_reduction_pct=0.872101 moved_to_echo=16491 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[552, 479, 552, 526, 492, 506, 502, 482, 20, 0, 0, 0, 0, 0, 0, 0, 524, 492, 516, 478, 525, 507, 470, 526, 0, 0, 0, 0, 0, 0, 0, 0, 513, 500, 507, 476, 525, 513, 515, 544, 0, 0, 0, 0, 0, 0, 0, 0, 500, 519, 508, 503, 515, 519, 526, 484, 18, 15, 0, 0, 0, 0, 0, 0, 517, 520, 497, 483, 538, 548, 518, 481, 0, 0, 0, 0, 0, 0, 0, 0, 526, 513, 488, 488, 504, 541, 517, 513, 28, 7, 0, 0, 0, 0, 0, 0, 562, 553, 548, 532, 479, 496, 507, 490, 0, 0, 0, 0, 0, 0, 0, 0, 503, 482, 505, 508, 459, 525, 505, 493, 45, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4111, 4038, 4093, 4107, 4102, 4125, 4167, 4025] avg=4096.0000 max=4167 max_rank=6 min=4025 min_rank=7 imbalance_ratio=1.017334 max_load_reduction=28601 max_load_reduction_pct=0.872833 moved_to_echo=16419 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 551, 482, 530, 490, 502, 522, 515, 20, 0, 0, 0, 0, 0, 0, 0, 525, 505, 543, 507, 516, 510, 492, 489, 0, 0, 0, 0, 0, 0, 0, 0, 555, 534, 512, 454, 505, 547, 508, 538, 0, 0, 0, 0, 0, 0, 0, 0, 520, 488, 515, 516, 513, 495, 487, 508, 18, 15, 0, 0, 0, 0, 0, 0, 508, 504, 491, 524, 494, 503, 526, 506, 0, 0, 0, 0, 0, 0, 0, 0, 508, 518, 496, 472, 521, 516, 521, 504, 28, 7, 0, 0, 0, 0, 0, 0, 504, 499, 505, 560, 491, 527, 488, 519, 0, 0, 0, 0, 0, 0, 0, 0, 545, 496, 488, 531, 478, 496, 504, 503, 43, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4087, 4153, 4075, 4056, 4091, 4093, 4084] avg=4096.0000 max=4153 max_rank=2 min=4056 min_rank=4 imbalance_ratio=1.013916 max_load_reduction=28615 max_load_reduction_pct=0.873260 moved_to_echo=16324 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[511, 527, 492, 488, 553, 516, 527, 483, 21, 0, 0, 0, 0, 0, 0, 0, 498, 526, 509, 512, 514, 503, 494, 530, 0, 0, 0, 0, 0, 0, 0, 0, 511, 551, 500, 538, 528, 528, 498, 530, 0, 0, 0, 0, 0, 0, 0, 0, 531, 482, 497, 494, 536, 463, 507, 527, 19, 15, 0, 0, 0, 0, 0, 0, 555, 484, 525, 473, 511, 515, 486, 505, 0, 0, 0, 0, 0, 0, 0, 0, 491, 495, 494, 476, 525, 541, 491, 531, 29, 8, 0, 0, 0, 0, 0, 0, 516, 537, 506, 478, 496, 510, 487, 505, 0, 0, 0, 0, 0, 0, 0, 0, 514, 503, 535, 511, 508, 516, 511, 490, 51, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4086, 4184, 4071, 4054, 4081, 4035, 4139] avg=4096.0000 max=4184 max_rank=2 min=4035 min_rank=6 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[525, 489, 529, 524, 491, 494, 513, 502, 21, 0, 0, 0, 0, 0, 0, 0, 503, 497, 528, 560, 525, 507, 511, 552, 0, 0, 0, 0, 0, 0, 0, 0, 490, 527, 527, 482, 507, 480, 484, 496, 0, 0, 0, 0, 0, 0, 0, 0, 482, 465, 542, 558, 519, 474, 521, 506, 19, 16, 0, 0, 0, 0, 0, 0, 530, 549, 515, 520, 546, 535, 459, 543, 0, 0, 0, 0, 0, 0, 0, 0, 504, 506, 525, 494, 490, 479, 471, 480, 28, 8, 0, 0, 0, 0, 0, 0, 532, 536, 515, 515, 492, 520, 516, 521, 0, 0, 0, 0, 0, 0, 0, 0, 512, 512, 512, 476, 503, 532, 462, 518, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4088, 4183, 3993, 4102, 4197, 3985, 4147, 4073] avg=4096.0000 max=4197 max_rank=4 min=3985 min_rank=5 imbalance_ratio=1.024658 max_load_reduction=28571 max_load_reduction_pct=0.871918 moved_to_echo=16402 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[517, 513, 475, 494, 493, 500, 493, 492, 26, 0, 0, 0, 0, 0, 0, 0, 521, 559, 488, 513, 521, 531, 510, 528, 0, 0, 0, 0, 0, 0, 0, 0, 574, 526, 527, 454, 496, 478, 508, 555, 0, 0, 0, 0, 0, 0, 0, 0, 525, 539, 468, 522, 520, 478, 469, 507, 21, 20, 0, 0, 0, 0, 0, 0, 490, 523, 524, 521, 480, 518, 464, 516, 4, 0, 0, 0, 0, 0, 0, 0, 521, 542, 532, 502, 523, 507, 499, 514, 31, 11, 0, 0, 0, 0, 0, 0, 480, 534, 535, 521, 469, 513, 523, 482, 0, 0, 0, 0, 0, 0, 0, 0, 513, 501, 508, 540, 499, 493, 517, 511, 46, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4003, 4171, 4118, 4069, 4040, 4182, 4057, 4128] avg=4096.0000 max=4182 max_rank=5 min=4003 min_rank=0 imbalance_ratio=1.020996 max_load_reduction=28586 max_load_reduction_pct=0.872375 moved_to_echo=16407 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[513, 549, 499, 495, 525, 549, 478, 532, 19, 0, 0, 0, 0, 0, 0, 0, 532, 524, 527, 482, 498, 536, 468, 503, 0, 0, 0, 0, 0, 0, 0, 0, 545, 468, 513, 468, 514, 568, 515, 535, 0, 0, 0, 0, 0, 0, 0, 0, 477, 492, 502, 499, 505, 500, 521, 522, 18, 15, 0, 0, 0, 0, 0, 0, 510, 533, 511, 507, 510, 485, 522, 529, 0, 0, 0, 0, 0, 0, 0, 0, 455, 528, 510, 530, 521, 522, 495, 482, 27, 7, 0, 0, 0, 0, 0, 0, 512, 506, 497, 507, 467, 575, 503, 525, 0, 0, 0, 0, 0, 0, 0, 0, 514, 515, 503, 479, 485, 504, 508, 534, 44, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4159, 4070, 4126, 4051, 4107, 4077, 4092, 4086] avg=4096.0000 max=4159 max_rank=0 min=4051 min_rank=3 imbalance_ratio=1.015381 max_load_reduction=28609 max_load_reduction_pct=0.873077 moved_to_echo=16362 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4224, 4224, 4224, 3968, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 3968, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4096, 4096, 4224, 4096, 3968, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4224, 3968, 4096, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4096, 4224, 4224, 3968, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[498, 536, 515, 528, 560, 530, 512, 517, 571, 492, 517, 479, 517, 494, 498, 502, 520, 525, 540, 503, 521, 508, 524, 480, 540, 485, 499, 482, 521, 489, 520, 498, 515, 496, 502, 545, 510, 482, 512, 552, 518, 533, 464, 512, 482, 524, 507, 482, 498, 532, 505, 497, 515, 505, 499, 534, 525, 512, 497, 561, 507, 491, 510, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[516, 487, 527, 522, 539, 517, 509, 496, 479, 517, 510, 517, 518, 466, 502, 517, 493, 536, 535, 504, 492, 504, 519, 541, 488, 513, 512, 530, 560, 536, 544, 514, 475, 557, 524, 486, 522, 488, 507, 521, 492, 483, 501, 512, 518, 483, 506, 512, 513, 531, 516, 490, 502, 508, 527, 530, 521, 504, 488, 488, 538, 538, 556, 471] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[496, 513, 505, 488, 472, 522, 524, 519, 542, 514, 493, 514, 529, 534, 499, 499, 527, 485, 510, 515, 517, 515, 501, 468, 524, 520, 464, 488, 515, 504, 532, 539, 491, 533, 513, 524, 508, 519, 506, 518, 495, 541, 503, 530, 526, 510, 489, 503, 510, 559, 489, 550, 500, 529, 480, 541, 502, 518, 540, 516, 502, 511, 521, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[489, 486, 535, 491, 513, 471, 435, 513, 500, 542, 528, 531, 510, 477, 566, 486, 527, 497, 491, 524, 508, 544, 541, 478, 505, 524, 520, 555, 536, 520, 472, 513, 537, 522, 488, 508, 491, 560, 519, 512, 525, 491, 491, 489, 498, 483, 514, 507, 514, 515, 537, 538, 491, 537, 516, 535, 561, 513, 493, 498, 498, 522, 533, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[511, 545, 538, 502, 536, 522, 523, 538, 547, 520, 530, 491, 467, 513, 519, 507, 500, 488, 524, 530, 490, 526, 517, 523, 524, 558, 476, 513, 535, 472, 486, 517, 514, 534, 503, 512, 480, 544, 481, 529, 513, 524, 469, 523, 541, 496, 529, 530, 476, 535, 455, 481, 482, 559, 528, 483, 495, 495, 533, 485, 535, 498, 501, 517] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[509, 493, 510, 547, 491, 529, 481, 508, 565, 499, 527, 515, 505, 537, 513, 543, 507, 478, 525, 513, 546, 534, 485, 535, 523, 519, 487, 487, 542, 506, 522, 508, 544, 483, 496, 497, 474, 524, 526, 506, 526, 495, 542, 508, 519, 462, 477, 555, 529, 515, 461, 519, 498, 515, 567, 542, 469, 455, 503, 541, 478, 544, 507, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[573, 537, 472, 515, 500, 540, 486, 521, 506, 517, 527, 516, 524, 476, 514, 461, 521, 527, 520, 505, 524, 510, 485, 492, 510, 526, 535, 517, 514, 498, 534, 508, 518, 517, 500, 494, 538, 470, 497, 449, 523, 478, 485, 497, 533, 518, 534, 513, 522, 515, 549, 512, 526, 518, 567, 504, 508, 516, 512, 519, 544, 485, 492, 504] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[523, 492, 526, 510, 506, 521, 517, 493, 514, 500, 524, 542, 492, 519, 505, 503, 517, 537, 518, 530, 513, 529, 529, 523, 465, 490, 549, 497, 521, 498, 536, 496, 552, 503, 519, 516, 505, 529, 492, 516, 508, 496, 491, 534, 548, 536, 474, 506, 528, 510, 547, 524, 545, 502, 515, 486, 474, 524, 497, 495, 517, 470, 497, 477] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[498, 536, 515, 528, 560, 527, 512, 517, 0, 0, 0, 0, 0, 0, 0, 0, 567, 492, 517, 479, 517, 494, 498, 502, 0, 0, 0, 0, 0, 0, 0, 0, 520, 525, 540, 503, 521, 493, 524, 480, 0, 0, 0, 0, 0, 0, 0, 0, 540, 485, 499, 482, 514, 489, 520, 498, 0, 0, 0, 0, 0, 0, 0, 0, 515, 496, 502, 545, 510, 482, 512, 552, 4, 3, 0, 0, 0, 0, 0, 0, 518, 533, 464, 512, 482, 524, 507, 482, 32, 0, 0, 0, 0, 0, 0, 0, 498, 497, 505, 497, 515, 505, 499, 534, 0, 0, 0, 0, 0, 0, 0, 0, 525, 512, 497, 561, 507, 491, 510, 523, 15, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4193, 4066, 4106, 4027, 4121, 4054, 4050, 4151] avg=4096.0000 max=4193 max_rank=0 min=4027 min_rank=3 imbalance_ratio=1.023682 max_load_reduction=28575 max_load_reduction_pct=0.872040 moved_to_echo=16376 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[496, 513, 505, 488, 472, 519, 524, 519, 0, 0, 0, 0, 0, 0, 0, 0, 539, 514, 493, 514, 529, 534, 499, 499, 0, 0, 0, 0, 0, 0, 0, 0, 527, 485, 510, 515, 517, 500, 501, 468, 0, 0, 0, 0, 0, 0, 0, 0, 524, 520, 464, 488, 508, 504, 532, 539, 0, 0, 0, 0, 0, 0, 0, 0, 491, 533, 513, 524, 508, 519, 506, 518, 3, 3, 0, 0, 0, 0, 0, 0, 495, 541, 503, 530, 526, 510, 489, 503, 34, 0, 0, 0, 0, 0, 0, 0, 510, 522, 489, 550, 500, 529, 480, 541, 0, 0, 0, 0, 0, 0, 0, 0, 502, 518, 540, 516, 502, 511, 521, 504, 15, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4036, 4121, 4023, 4079, 4118, 4131, 4121, 4139] avg=4096.0000 max=4139 max_rank=7 min=4023 min_rank=2 imbalance_ratio=1.010498 max_load_reduction=28629 max_load_reduction_pct=0.873688 moved_to_echo=16509 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[516, 487, 527, 522, 539, 514, 509, 496, 0, 0, 0, 0, 0, 0, 0, 0, 476, 517, 510, 517, 518, 466, 502, 517, 0, 0, 0, 0, 0, 0, 0, 0, 493, 536, 535, 504, 492, 489, 519, 541, 0, 0, 0, 0, 0, 0, 0, 0, 488, 513, 512, 530, 553, 536, 544, 514, 0, 0, 0, 0, 0, 0, 0, 0, 475, 557, 524, 486, 522, 488, 507, 521, 3, 3, 0, 0, 0, 0, 0, 0, 492, 483, 501, 512, 518, 483, 506, 512, 32, 0, 0, 0, 0, 0, 0, 0, 513, 496, 516, 490, 502, 508, 527, 530, 0, 0, 0, 0, 0, 0, 0, 0, 521, 504, 488, 488, 538, 538, 556, 471, 15, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4110, 4023, 4109, 4190, 4086, 4039, 4082, 4129] avg=4096.0000 max=4190 max_rank=3 min=4023 min_rank=1 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16336 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[509, 493, 510, 547, 491, 526, 481, 508, 0, 0, 0, 0, 0, 0, 0, 0, 561, 499, 527, 515, 505, 537, 513, 543, 0, 0, 0, 0, 0, 0, 0, 0, 507, 478, 525, 513, 546, 518, 485, 535, 0, 0, 0, 0, 0, 0, 0, 0, 523, 519, 487, 487, 535, 506, 522, 508, 0, 0, 0, 0, 0, 0, 0, 0, 544, 483, 496, 497, 474, 524, 526, 506, 4, 3, 0, 0, 0, 0, 0, 0, 526, 495, 542, 508, 519, 462, 477, 555, 31, 0, 0, 0, 0, 0, 0, 0, 529, 481, 461, 519, 498, 515, 567, 542, 0, 0, 0, 0, 0, 0, 0, 0, 469, 455, 503, 541, 478, 544, 507, 502, 16, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4065, 4200, 4107, 4087, 4057, 4115, 4112, 4025] avg=4096.0000 max=4200 max_rank=1 min=4025 min_rank=7 imbalance_ratio=1.025391 max_load_reduction=28568 max_load_reduction_pct=0.871826 moved_to_echo=16309 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[489, 486, 535, 491, 513, 468, 435, 513, 0, 0, 0, 0, 0, 0, 0, 0, 497, 542, 528, 531, 510, 477, 566, 486, 0, 0, 0, 0, 0, 0, 0, 0, 527, 497, 491, 524, 508, 528, 541, 478, 0, 0, 0, 0, 0, 0, 0, 0, 505, 524, 520, 555, 529, 520, 472, 513, 0, 0, 0, 0, 0, 0, 0, 0, 537, 522, 488, 508, 491, 560, 519, 512, 3, 3, 0, 0, 0, 0, 0, 0, 525, 491, 491, 489, 498, 483, 514, 507, 31, 0, 0, 0, 0, 0, 0, 0, 514, 481, 537, 538, 491, 537, 516, 535, 0, 0, 0, 0, 0, 0, 0, 0, 561, 513, 493, 498, 498, 522, 533, 504, 16, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3930, 4137, 4094, 4138, 4143, 4029, 4149, 4148] avg=4096.0000 max=4149 max_rank=6 min=3930 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16469 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[511, 545, 538, 502, 536, 519, 523, 538, 0, 0, 0, 0, 0, 0, 0, 0, 544, 520, 530, 491, 467, 513, 519, 507, 0, 0, 0, 0, 0, 0, 0, 0, 500, 488, 524, 530, 490, 511, 517, 523, 0, 0, 0, 0, 0, 0, 0, 0, 524, 558, 476, 513, 528, 472, 486, 517, 0, 0, 0, 0, 0, 0, 0, 0, 514, 534, 503, 512, 480, 544, 481, 529, 3, 3, 0, 0, 0, 0, 0, 0, 513, 524, 469, 523, 541, 496, 529, 530, 33, 0, 0, 0, 0, 0, 0, 0, 476, 499, 455, 481, 482, 559, 528, 483, 0, 0, 0, 0, 0, 0, 0, 0, 495, 495, 533, 485, 535, 498, 501, 517, 15, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4212, 4091, 4083, 4074, 4103, 4158, 3963, 4084] avg=4096.0000 max=4212 max_rank=0 min=3963 min_rank=6 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16308 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[573, 537, 472, 515, 500, 533, 486, 521, 0, 0, 0, 0, 0, 0, 0, 0, 499, 517, 527, 516, 524, 476, 514, 461, 0, 0, 0, 0, 0, 0, 0, 0, 521, 527, 520, 505, 524, 491, 485, 492, 0, 0, 0, 0, 0, 0, 0, 0, 510, 526, 535, 517, 500, 498, 534, 508, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 500, 494, 538, 470, 497, 449, 7, 7, 5, 0, 0, 0, 0, 0, 523, 478, 485, 497, 533, 518, 534, 513, 37, 0, 0, 0, 0, 0, 0, 0, 522, 470, 549, 512, 526, 518, 567, 504, 0, 0, 0, 0, 0, 0, 0, 0, 508, 516, 512, 519, 544, 485, 492, 504, 19, 9, 8, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4137, 4034, 4065, 4128, 4002, 4118, 4168, 4116] avg=4096.0000 max=4168 max_rank=6 min=4002 min_rank=4 imbalance_ratio=1.017578 max_load_reduction=28600 max_load_reduction_pct=0.872803 moved_to_echo=16404 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[523, 492, 526, 510, 506, 518, 517, 493, 0, 0, 0, 0, 0, 0, 0, 0, 511, 500, 524, 542, 492, 519, 505, 503, 0, 0, 0, 0, 0, 0, 0, 0, 517, 537, 518, 530, 513, 514, 529, 523, 0, 0, 0, 0, 0, 0, 0, 0, 465, 490, 549, 497, 514, 498, 536, 496, 0, 0, 0, 0, 0, 0, 0, 0, 552, 503, 519, 516, 505, 529, 492, 516, 3, 3, 0, 0, 0, 0, 0, 0, 508, 496, 491, 534, 548, 536, 474, 506, 31, 0, 0, 0, 0, 0, 0, 0, 528, 476, 547, 524, 545, 502, 515, 486, 0, 0, 0, 0, 0, 0, 0, 0, 474, 524, 497, 495, 517, 470, 497, 477, 15, 7, 3, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4085, 4096, 4181, 4045, 4138, 4124, 4123, 3976] avg=4096.0000 max=4181 max_rank=2 min=3976 min_rank=7 imbalance_ratio=1.020752 max_load_reduction=28587 max_load_reduction_pct=0.872406 moved_to_echo=16361 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 3968, 4096, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 3968, 4224, 4224, 4096, 4096, 4224, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4096, 4224, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4224, 4224, 4096, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[4096, 4224, 4096, 4096, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[541, 475, 539, 507, 516, 519, 484, 502, 528, 523, 541, 548, 506, 515, 497, 541, 520, 504, 509, 506, 523, 524, 484, 557, 494, 509, 519, 482, 462, 487, 501, 504, 496, 530, 517, 524, 516, 528, 531, 487, 512, 504, 514, 494, 526, 490, 528, 535, 507, 519, 517, 487, 507, 533, 517, 536, 467, 539, 479, 547, 489, 523, 497, 505] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[491, 504, 522, 508, 502, 517, 503, 532, 545, 540, 505, 544, 542, 532, 539, 520, 515, 495, 524, 517, 511, 580, 495, 520, 534, 500, 486, 498, 502, 537, 511, 524, 479, 478, 500, 520, 527, 509, 495, 483, 497, 510, 549, 464, 472, 533, 503, 485, 489, 534, 533, 476, 508, 523, 480, 516, 480, 503, 520, 520, 522, 484, 558, 523] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[532, 501, 545, 518, 463, 512, 538, 510, 530, 539, 488, 516, 505, 515, 514, 514, 547, 533, 553, 536, 498, 518, 514, 516, 530, 514, 489, 509, 492, 531, 501, 502, 477, 508, 527, 512, 512, 557, 507, 506, 537, 503, 517, 535, 500, 500, 492, 507, 490, 518, 543, 477, 486, 517, 507, 518, 484, 512, 498, 491, 489, 528, 464, 526] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[519, 512, 533, 503, 513, 531, 526, 508, 505, 509, 520, 532, 536, 531, 518, 489, 529, 528, 517, 509, 494, 503, 514, 502, 497, 484, 521, 536, 496, 508, 470, 504, 477, 496, 508, 491, 516, 529, 511, 527, 560, 514, 529, 520, 496, 501, 492, 505, 506, 473, 507, 532, 496, 515, 498, 523, 536, 566, 510, 510, 505, 500, 519, 503] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[483, 526, 542, 541, 491, 527, 541, 527, 541, 490, 509, 559, 519, 527, 499, 510, 477, 498, 515, 537, 538, 540, 527, 502, 492, 467, 514, 539, 493, 498, 527, 487, 475, 508, 493, 509, 487, 524, 512, 519, 509, 520, 533, 464, 466, 525, 500, 508, 482, 518, 542, 476, 479, 516, 505, 535, 484, 533, 525, 559, 488, 533, 550, 508] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[523, 510, 521, 515, 522, 555, 507, 511, 511, 520, 514, 536, 501, 508, 537, 507, 533, 497, 527, 515, 503, 508, 523, 514, 497, 500, 515, 497, 506, 521, 522, 525, 501, 519, 511, 491, 519, 502, 511, 527, 479, 494, 519, 510, 541, 470, 545, 530, 474, 479, 474, 538, 509, 542, 524, 528, 525, 506, 501, 477, 525, 508, 514, 474] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[522, 456, 532, 511, 507, 528, 560, 526, 490, 519, 563, 489, 495, 510, 517, 490, 501, 549, 526, 523, 518, 522, 472, 515, 521, 528, 472, 513, 543, 530, 487, 497, 509, 485, 490, 544, 501, 524, 495, 504, 501, 512, 529, 559, 493, 573, 524, 510, 508, 494, 514, 504, 527, 486, 509, 520, 503, 504, 532, 527, 463, 501, 515, 476] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[520, 529, 515, 539, 522, 488, 529, 512, 557, 515, 507, 517, 538, 467, 501, 497, 511, 490, 524, 510, 536, 524, 556, 508, 497, 528, 532, 513, 539, 538, 519, 543, 453, 517, 509, 502, 493, 494, 470, 521, 501, 501, 518, 484, 491, 494, 542, 470, 519, 519, 524, 549, 526, 525, 514, 477, 502, 511, 512, 471, 508, 501, 469, 560] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[541, 475, 503, 507, 516, 519, 484, 502, 0, 0, 0, 0, 0, 0, 0, 0, 528, 523, 541, 495, 506, 515, 497, 541, 0, 0, 0, 0, 0, 0, 0, 0, 520, 504, 509, 506, 523, 480, 484, 557, 0, 0, 0, 0, 0, 0, 0, 0, 494, 509, 519, 482, 462, 487, 501, 504, 22, 6, 0, 0, 0, 0, 0, 0, 496, 530, 517, 524, 516, 528, 531, 487, 47, 0, 0, 0, 0, 0, 0, 0, 512, 504, 514, 494, 526, 490, 528, 535, 16, 0, 0, 0, 0, 0, 0, 0, 507, 519, 517, 487, 507, 533, 517, 536, 18, 0, 0, 0, 0, 0, 0, 0, 467, 539, 479, 547, 489, 523, 497, 505, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4047, 4146, 4083, 3986, 4176, 4119, 4141, 4070] avg=4096.0000 max=4176 max_rank=4 min=3986 min_rank=3 imbalance_ratio=1.019531 max_load_reduction=28592 max_load_reduction_pct=0.872559 moved_to_echo=16506 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[519, 512, 497, 503, 513, 531, 526, 508, 0, 0, 0, 0, 0, 0, 0, 0, 505, 509, 520, 480, 536, 531, 518, 489, 0, 0, 0, 0, 0, 0, 0, 0, 529, 528, 517, 509, 494, 461, 514, 502, 0, 0, 0, 0, 0, 0, 0, 0, 497, 484, 521, 536, 496, 508, 470, 504, 21, 6, 0, 0, 0, 0, 0, 0, 477, 496, 508, 491, 516, 529, 511, 527, 46, 0, 0, 0, 0, 0, 0, 0, 560, 514, 529, 520, 496, 501, 492, 505, 16, 0, 0, 0, 0, 0, 0, 0, 506, 473, 507, 532, 496, 515, 498, 523, 18, 0, 0, 0, 0, 0, 0, 0, 536, 566, 510, 510, 505, 500, 519, 503, 21, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4109, 4088, 4054, 4043, 4101, 4133, 4068, 4172] avg=4096.0000 max=4172 max_rank=7 min=4043 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16474 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[491, 504, 488, 508, 502, 517, 503, 532, 0, 0, 0, 0, 0, 0, 0, 0, 545, 540, 505, 491, 542, 532, 539, 520, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 524, 517, 511, 530, 495, 520, 0, 0, 0, 0, 0, 0, 0, 0, 534, 500, 486, 498, 502, 537, 511, 524, 25, 6, 0, 0, 0, 0, 0, 0, 479, 478, 500, 520, 527, 509, 495, 483, 47, 0, 0, 0, 0, 0, 0, 0, 497, 510, 549, 464, 472, 533, 503, 485, 15, 0, 0, 0, 0, 0, 0, 0, 489, 534, 533, 476, 508, 523, 480, 516, 17, 0, 0, 0, 0, 0, 0, 0, 480, 503, 520, 520, 522, 484, 558, 523, 25, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4045, 4214, 4107, 4123, 4038, 4028, 4076, 4137] avg=4096.0000 max=4214 max_rank=1 min=4028 min_rank=5 imbalance_ratio=1.028809 max_load_reduction=28554 max_load_reduction_pct=0.871399 moved_to_echo=16279 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[532, 501, 509, 518, 463, 512, 538, 510, 0, 0, 0, 0, 0, 0, 0, 0, 530, 539, 488, 466, 505, 515, 514, 514, 0, 0, 0, 0, 0, 0, 0, 0, 547, 533, 553, 536, 498, 474, 514, 516, 0, 0, 0, 0, 0, 0, 0, 0, 530, 514, 489, 509, 492, 531, 501, 502, 22, 6, 0, 0, 0, 0, 0, 0, 477, 508, 527, 512, 512, 557, 507, 506, 44, 0, 0, 0, 0, 0, 0, 0, 537, 503, 517, 535, 500, 500, 492, 507, 16, 0, 0, 0, 0, 0, 0, 0, 490, 518, 543, 477, 486, 517, 507, 518, 18, 0, 0, 0, 0, 0, 0, 0, 484, 512, 498, 491, 489, 528, 464, 526, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4083, 4071, 4171, 4096, 4150, 4107, 4074, 4016] avg=4096.0000 max=4171 max_rank=2 min=4016 min_rank=7 imbalance_ratio=1.018311 max_load_reduction=28597 max_load_reduction_pct=0.872711 moved_to_echo=16347 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[483, 526, 506, 541, 491, 527, 541, 527, 0, 0, 0, 0, 0, 0, 0, 0, 541, 490, 509, 505, 519, 527, 499, 510, 0, 0, 0, 0, 0, 0, 0, 0, 477, 498, 515, 537, 538, 494, 527, 502, 0, 0, 0, 0, 0, 0, 0, 0, 492, 467, 514, 539, 493, 498, 527, 487, 23, 6, 0, 0, 0, 0, 0, 0, 475, 508, 493, 509, 487, 524, 512, 519, 48, 0, 0, 0, 0, 0, 0, 0, 509, 520, 533, 464, 466, 525, 500, 508, 16, 0, 0, 0, 0, 0, 0, 0, 482, 518, 542, 476, 479, 516, 505, 535, 18, 0, 0, 0, 0, 0, 0, 0, 484, 533, 525, 559, 488, 533, 550, 508, 23, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4142, 4100, 4088, 4046, 4075, 4041, 4071, 4205] avg=4096.0000 max=4205 max_rank=7 min=4041 min_rank=5 imbalance_ratio=1.026611 max_load_reduction=28563 max_load_reduction_pct=0.871674 moved_to_echo=16392 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[523, 510, 487, 515, 522, 555, 507, 511, 0, 0, 0, 0, 0, 0, 0, 0, 511, 520, 514, 484, 501, 508, 537, 507, 0, 0, 0, 0, 0, 0, 0, 0, 533, 497, 527, 515, 503, 464, 523, 514, 0, 0, 0, 0, 0, 0, 0, 0, 497, 500, 515, 497, 506, 521, 522, 525, 22, 6, 0, 0, 0, 0, 0, 0, 501, 519, 511, 491, 519, 502, 511, 527, 46, 0, 0, 0, 0, 0, 0, 0, 479, 494, 519, 510, 541, 470, 545, 530, 15, 0, 0, 0, 0, 0, 0, 0, 474, 479, 474, 538, 509, 542, 524, 528, 17, 0, 0, 0, 0, 0, 0, 0, 525, 506, 501, 477, 525, 508, 514, 474, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4130, 4082, 4076, 4111, 4127, 4103, 4085, 4054] avg=4096.0000 max=4130 max_rank=0 min=4054 min_rank=7 imbalance_ratio=1.008301 max_load_reduction=28638 max_load_reduction_pct=0.873962 moved_to_echo=16369 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[522, 456, 482, 511, 507, 528, 560, 526, 0, 0, 0, 0, 0, 0, 0, 0, 490, 519, 563, 434, 495, 510, 517, 490, 0, 0, 0, 0, 0, 0, 0, 0, 501, 549, 526, 523, 518, 470, 472, 515, 0, 0, 0, 0, 0, 0, 0, 0, 521, 528, 472, 513, 543, 530, 487, 497, 26, 9, 0, 0, 0, 0, 0, 0, 509, 485, 490, 544, 501, 524, 495, 504, 46, 0, 0, 0, 0, 0, 0, 0, 501, 512, 529, 559, 493, 573, 524, 510, 20, 0, 0, 0, 0, 0, 0, 0, 508, 494, 514, 504, 527, 486, 509, 520, 21, 0, 0, 0, 0, 0, 0, 0, 503, 504, 532, 527, 463, 501, 515, 476, 26, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4092, 4018, 4074, 4126, 4098, 4221, 4083, 4056] avg=4096.0000 max=4221 max_rank=5 min=4018 min_rank=1 imbalance_ratio=1.030518 max_load_reduction=28547 max_load_reduction_pct=0.871185 moved_to_echo=16458 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[520, 529, 481, 539, 522, 488, 529, 512, 0, 0, 0, 0, 0, 0, 0, 0, 557, 515, 507, 467, 538, 467, 501, 497, 0, 0, 0, 0, 0, 0, 0, 0, 511, 490, 524, 510, 536, 480, 556, 508, 0, 0, 0, 0, 0, 0, 0, 0, 497, 528, 532, 513, 539, 538, 519, 543, 22, 6, 0, 0, 0, 0, 0, 0, 453, 517, 509, 502, 493, 494, 470, 521, 44, 0, 0, 0, 0, 0, 0, 0, 501, 501, 518, 484, 491, 494, 542, 470, 15, 0, 0, 0, 0, 0, 0, 0, 519, 519, 524, 549, 526, 525, 514, 477, 17, 0, 0, 0, 0, 0, 0, 0, 502, 511, 512, 471, 508, 501, 469, 560, 22, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4120, 4049, 4115, 4237, 4003, 4016, 4170, 4058] avg=4096.0000 max=4237 max_rank=3 min=4003 min_rank=4 imbalance_ratio=1.034424 max_load_reduction=28531 max_load_reduction_pct=0.870697 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 3968, 4224, 4096, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4224, 3840, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4224, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4096, 4224, 4096, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 3968, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4224, 4096, 256, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33152 observed_loss=-384 tokens_per_expert=[3968, 4096, 4096, 4096, 4096, 4224, 4096, 4096, 384, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33152 slack_slots=32384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[488, 509, 512, 525, 513, 516, 509, 503, 500, 577, 495, 496, 505, 530, 553, 533, 500, 523, 525, 507, 523, 487, 537, 512, 528, 529, 502, 507, 495, 523, 493, 506, 520, 505, 525, 461, 561, 531, 494, 471, 505, 521, 521, 531, 523, 526, 523, 530, 493, 490, 489, 523, 509, 511, 467, 506, 534, 505, 499, 502, 512, 497, 500, 522] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[522, 522, 549, 538, 523, 491, 526, 525, 509, 540, 479, 501, 467, 509, 543, 487, 517, 563, 549, 540, 499, 524, 505, 482, 513, 478, 479, 496, 515, 464, 554, 500, 532, 501, 516, 511, 487, 499, 482, 486, 558, 512, 506, 493, 496, 486, 554, 546, 493, 486, 525, 530, 530, 523, 561, 505, 492, 501, 477, 497, 510, 511, 507, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[556, 541, 505, 555, 514, 503, 528, 521, 507, 529, 481, 504, 479, 508, 520, 517, 526, 441, 537, 493, 491, 519, 532, 524, 522, 512, 508, 526, 529, 494, 525, 521, 491, 514, 513, 554, 494, 499, 498, 516, 495, 519, 465, 509, 519, 558, 506, 531, 481, 541, 531, 555, 472, 522, 517, 517, 508, 517, 483, 490, 502, 494, 483, 506] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[512, 493, 501, 528, 524, 487, 543, 504, 518, 522, 538, 499, 516, 526, 514, 511, 495, 494, 510, 492, 509, 498, 497, 490, 501, 448, 482, 514, 527, 544, 491, 511, 567, 507, 549, 536, 460, 536, 518, 528, 513, 496, 504, 528, 499, 524, 564, 519, 514, 510, 526, 507, 513, 484, 527, 491, 543, 522, 521, 499, 510, 485, 514, 515] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[516, 523, 510, 516, 516, 516, 511, 525, 502, 501, 531, 512, 531, 520, 502, 480, 501, 532, 486, 488, 483, 489, 476, 507, 538, 522, 551, 528, 473, 539, 498, 509, 492, 475, 519, 503, 523, 490, 491, 502, 540, 497, 485, 522, 530, 511, 523, 518, 514, 561, 514, 530, 538, 532, 518, 506, 539, 509, 486, 471, 518, 518, 531, 530] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[501, 542, 502, 522, 505, 539, 508, 530, 528, 519, 534, 536, 498, 534, 531, 512, 519, 468, 496, 512, 498, 470, 518, 510, 519, 480, 526, 533, 506, 546, 525, 512, 524, 523, 529, 533, 510, 497, 514, 517, 506, 495, 509, 509, 531, 529, 498, 494, 513, 505, 546, 487, 501, 486, 509, 483, 508, 514, 501, 515, 525, 505, 489, 484] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[528, 509, 517, 509, 517, 504, 512, 485, 528, 513, 502, 500, 515, 529, 497, 456, 496, 495, 530, 528, 513, 482, 484, 554, 472, 508, 495, 524, 499, 530, 501, 524, 514, 533, 484, 549, 572, 502, 548, 498, 541, 528, 506, 535, 530, 483, 459, 543, 510, 506, 492, 546, 496, 549, 524, 507, 527, 494, 527, 492, 477, 523, 515, 502] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[516, 495, 512, 501, 508, 543, 482, 519, 515, 500, 512, 510, 491, 528, 530, 503, 508, 504, 527, 498, 514, 526, 516, 525, 515, 494, 523, 494, 540, 535, 493, 526, 508, 476, 520, 539, 530, 503, 481, 545, 563, 491, 520, 553, 512, 504, 513, 508, 479, 512, 508, 523, 487, 515, 514, 530, 555, 479, 509, 468, 515, 471, 518, 516] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[488, 509, 512, 494, 513, 516, 509, 503, 0, 0, 0, 0, 0, 0, 0, 0, 500, 571, 495, 496, 505, 530, 553, 533, 0, 0, 0, 0, 0, 0, 0, 0, 500, 523, 525, 507, 523, 487, 537, 512, 27, 6, 0, 0, 0, 0, 0, 0, 528, 529, 502, 507, 495, 523, 493, 506, 3, 2, 0, 0, 0, 0, 0, 0, 520, 505, 525, 458, 561, 531, 494, 471, 0, 0, 0, 0, 0, 0, 0, 0, 470, 521, 521, 531, 523, 526, 523, 530, 0, 0, 0, 0, 0, 0, 0, 0, 493, 490, 489, 521, 509, 511, 467, 506, 0, 0, 0, 0, 0, 0, 0, 0, 534, 505, 499, 502, 512, 497, 500, 522, 35, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4044, 4183, 4147, 4088, 4065, 4145, 3986, 4110] avg=4096.0000 max=4183 max_rank=1 min=3986 min_rank=6 imbalance_ratio=1.021240 max_load_reduction=28585 max_load_reduction_pct=0.872345 moved_to_echo=16306 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[512, 493, 501, 497, 524, 487, 543, 504, 0, 0, 0, 0, 0, 0, 0, 0, 518, 517, 538, 499, 516, 526, 514, 511, 0, 0, 0, 0, 0, 0, 0, 0, 495, 494, 510, 492, 509, 498, 497, 490, 27, 5, 0, 0, 0, 0, 0, 0, 501, 448, 482, 514, 527, 544, 491, 511, 3, 2, 0, 0, 0, 0, 0, 0, 567, 507, 549, 533, 460, 536, 518, 528, 0, 0, 0, 0, 0, 0, 0, 0, 477, 496, 504, 528, 499, 524, 564, 519, 0, 0, 0, 0, 0, 0, 0, 0, 514, 510, 526, 505, 513, 484, 527, 491, 0, 0, 0, 0, 0, 0, 0, 0, 543, 522, 521, 499, 510, 485, 514, 515, 36, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4139, 4017, 4023, 4198, 4111, 4070, 4149] avg=4096.0000 max=4198 max_rank=4 min=4017 min_rank=2 imbalance_ratio=1.024902 max_load_reduction=28570 max_load_reduction_pct=0.871887 moved_to_echo=16528 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[501, 542, 502, 491, 505, 539, 508, 530, 0, 0, 0, 0, 0, 0, 0, 0, 528, 514, 534, 536, 498, 534, 531, 512, 0, 0, 0, 0, 0, 0, 0, 0, 519, 468, 496, 512, 498, 470, 518, 510, 27, 5, 0, 0, 0, 0, 0, 0, 519, 480, 526, 533, 506, 546, 525, 512, 3, 2, 0, 0, 0, 0, 0, 0, 524, 523, 529, 530, 510, 497, 514, 517, 0, 0, 0, 0, 0, 0, 0, 0, 471, 495, 509, 509, 531, 529, 498, 494, 0, 0, 0, 0, 0, 0, 0, 0, 513, 505, 546, 485, 501, 486, 509, 483, 0, 0, 0, 0, 0, 0, 0, 0, 508, 514, 501, 515, 525, 505, 489, 484, 35, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4118, 4187, 4023, 4152, 4144, 4036, 4028, 4080] avg=4096.0000 max=4187 max_rank=1 min=4023 min_rank=2 imbalance_ratio=1.022217 max_load_reduction=28581 max_load_reduction_pct=0.872223 moved_to_echo=16288 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[522, 522, 549, 506, 523, 491, 526, 525, 0, 0, 0, 0, 0, 0, 0, 0, 509, 535, 479, 501, 467, 509, 543, 487, 0, 0, 0, 0, 0, 0, 0, 0, 517, 563, 549, 540, 499, 524, 505, 482, 28, 5, 0, 0, 0, 0, 0, 0, 513, 478, 479, 496, 515, 464, 554, 500, 3, 2, 0, 0, 0, 0, 0, 0, 532, 501, 516, 508, 487, 499, 482, 486, 0, 0, 0, 0, 0, 0, 0, 0, 519, 512, 506, 493, 496, 486, 554, 546, 0, 0, 0, 0, 0, 0, 0, 0, 493, 486, 525, 528, 530, 523, 561, 505, 0, 0, 0, 0, 0, 0, 0, 0, 492, 501, 477, 497, 510, 511, 507, 546, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4164, 4030, 4212, 4004, 4011, 4112, 4151, 4084] avg=4096.0000 max=4212 max_rank=2 min=4004 min_rank=3 imbalance_ratio=1.028320 max_load_reduction=28556 max_load_reduction_pct=0.871460 moved_to_echo=16358 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 523, 510, 485, 516, 516, 511, 525, 0, 0, 0, 0, 0, 0, 0, 0, 502, 496, 531, 512, 531, 520, 502, 480, 0, 0, 0, 0, 0, 0, 0, 0, 501, 532, 486, 488, 483, 489, 476, 507, 27, 5, 0, 0, 0, 0, 0, 0, 538, 522, 551, 528, 473, 539, 498, 509, 3, 2, 0, 0, 0, 0, 0, 0, 492, 475, 519, 500, 523, 490, 491, 502, 0, 0, 0, 0, 0, 0, 0, 0, 502, 497, 485, 522, 530, 511, 523, 518, 0, 0, 0, 0, 0, 0, 0, 0, 514, 561, 514, 528, 538, 532, 518, 506, 0, 0, 0, 0, 0, 0, 0, 0, 539, 509, 486, 471, 518, 518, 531, 530, 38, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4102, 4074, 3994, 4163, 3992, 4088, 4211, 4144] avg=4096.0000 max=4211 max_rank=6 min=3992 min_rank=4 imbalance_ratio=1.028076 max_load_reduction=28557 max_load_reduction_pct=0.871490 moved_to_echo=16435 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[556, 541, 505, 522, 514, 503, 528, 521, 0, 0, 0, 0, 0, 0, 0, 0, 507, 524, 481, 504, 479, 508, 520, 517, 0, 0, 0, 0, 0, 0, 0, 0, 526, 441, 537, 493, 491, 519, 532, 524, 29, 5, 0, 0, 0, 0, 0, 0, 522, 512, 508, 526, 529, 494, 525, 521, 4, 2, 0, 0, 0, 0, 0, 0, 491, 514, 513, 550, 494, 499, 498, 516, 0, 0, 0, 0, 0, 0, 0, 0, 461, 519, 465, 509, 519, 558, 506, 531, 0, 0, 0, 0, 0, 0, 0, 0, 481, 541, 531, 553, 472, 522, 517, 517, 0, 0, 0, 0, 0, 0, 0, 0, 508, 517, 483, 490, 502, 494, 483, 506, 34, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4190, 4040, 4097, 4143, 4075, 4068, 4134, 4021] avg=4096.0000 max=4190 max_rank=0 min=4021 min_rank=7 imbalance_ratio=1.022949 max_load_reduction=28578 max_load_reduction_pct=0.872131 moved_to_echo=16298 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[528, 509, 517, 471, 517, 504, 512, 485, 0, 0, 0, 0, 0, 0, 0, 0, 528, 504, 502, 500, 515, 529, 497, 456, 0, 0, 0, 0, 0, 0, 0, 0, 496, 495, 530, 528, 513, 482, 484, 554, 31, 9, 7, 0, 0, 0, 0, 0, 472, 508, 495, 524, 499, 530, 501, 524, 9, 8, 0, 0, 0, 0, 0, 0, 514, 533, 484, 533, 572, 502, 548, 498, 0, 0, 0, 0, 0, 0, 0, 0, 499, 528, 506, 535, 530, 483, 459, 543, 0, 0, 0, 0, 0, 0, 0, 0, 510, 506, 492, 538, 496, 549, 524, 507, 0, 0, 0, 0, 0, 0, 0, 0, 527, 494, 527, 492, 477, 523, 515, 502, 42, 7, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4043, 4031, 4129, 4070, 4184, 4083, 4122, 4106] avg=4096.0000 max=4184 max_rank=4 min=4031 min_rank=1 imbalance_ratio=1.021484 max_load_reduction=28584 max_load_reduction_pct=0.872314 moved_to_echo=16495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[516, 495, 512, 471, 508, 543, 482, 519, 0, 0, 0, 0, 0, 0, 0, 0, 515, 495, 512, 510, 491, 528, 530, 503, 0, 0, 0, 0, 0, 0, 0, 0, 508, 504, 527, 498, 514, 526, 516, 525, 26, 5, 0, 0, 0, 0, 0, 0, 515, 494, 523, 494, 540, 535, 493, 526, 3, 2, 0, 0, 0, 0, 0, 0, 508, 476, 520, 536, 530, 503, 481, 545, 0, 0, 0, 0, 0, 0, 0, 0, 524, 491, 520, 553, 512, 504, 513, 508, 0, 0, 0, 0, 0, 0, 0, 0, 479, 512, 508, 521, 487, 515, 514, 530, 0, 0, 0, 0, 0, 0, 0, 0, 555, 479, 509, 468, 515, 471, 518, 516, 39, 4, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4046, 4084, 4149, 4125, 4099, 4125, 4066, 4074] avg=4096.0000 max=4149 max_rank=2 min=4046 min_rank=0 imbalance_ratio=1.012939 max_load_reduction=28619 max_load_reduction_pct=0.873383 moved_to_echo=16364 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4096, 4224, 4224, 4224, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[3968, 4096, 4096, 4224, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 128, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4096, 3968, 4096, 4096, 4096, 4224, 384, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4224, 4224, 4224, 3968, 4224, 4224, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4224, 4096, 4096, 4096, 4096, 4224, 256, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[513, 509, 535, 526, 524, 501, 562, 518, 502, 504, 460, 488, 518, 531, 483, 519, 477, 513, 560, 575, 566, 471, 527, 519, 521, 485, 529, 466, 502, 544, 517, 502, 517, 504, 548, 500, 537, 481, 502, 499, 511, 492, 484, 500, 465, 541, 519, 512, 495, 530, 523, 508, 519, 536, 507, 533, 501, 500, 484, 521, 541, 510, 487, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[520, 534, 476, 508, 483, 557, 548, 534, 549, 543, 524, 530, 530, 513, 512, 501, 498, 532, 479, 534, 501, 490, 519, 525, 514, 474, 499, 479, 497, 486, 531, 498, 528, 518, 515, 525, 562, 492, 519, 487, 514, 467, 512, 507, 479, 515, 496, 479, 522, 514, 541, 544, 526, 496, 548, 502, 528, 490, 485, 505, 528, 485, 503, 518] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[567, 507, 503, 514, 510, 535, 487, 498, 521, 518, 513, 529, 521, 485, 573, 542, 496, 530, 509, 471, 514, 495, 506, 551, 522, 536, 518, 531, 536, 544, 494, 537, 490, 474, 499, 547, 492, 496, 515, 485, 503, 522, 509, 482, 497, 555, 527, 514, 524, 509, 529, 491, 494, 508, 536, 472, 508, 526, 501, 484, 521, 472, 479, 494] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[514, 530, 476, 505, 552, 551, 525, 490, 551, 544, 533, 533, 452, 527, 511, 575, 497, 494, 508, 502, 484, 534, 498, 485, 546, 532, 508, 504, 527, 487, 507, 498, 512, 517, 535, 535, 521, 505, 521, 518, 496, 509, 518, 483, 509, 494, 514, 502, 498, 494, 534, 523, 521, 525, 479, 507, 511, 492, 510, 472, 525, 512, 524, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[547, 543, 521, 513, 509, 517, 556, 485, 554, 536, 483, 476, 523, 501, 512, 508, 526, 541, 505, 559, 531, 501, 506, 518, 492, 502, 548, 515, 513, 481, 516, 491, 502, 467, 525, 513, 520, 513, 539, 521, 463, 456, 509, 511, 493, 481, 500, 489, 474, 492, 507, 491, 528, 523, 549, 541, 527, 500, 490, 521, 523, 503, 522, 546] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[510, 486, 486, 503, 485, 526, 514, 524, 488, 518, 486, 520, 530, 559, 502, 519, 524, 509, 526, 503, 527, 545, 481, 516, 523, 509, 490, 500, 500, 558, 514, 510, 520, 526, 537, 475, 493, 462, 507, 504, 519, 516, 509, 495, 522, 497, 482, 517, 552, 527, 543, 499, 490, 530, 501, 538, 519, 506, 513, 494, 545, 482, 523, 534] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[509, 514, 531, 535, 506, 529, 523, 534, 479, 518, 515, 510, 521, 497, 528, 524, 498, 517, 508, 491, 508, 492, 537, 516, 496, 499, 511, 510, 515, 492, 516, 504, 536, 495, 516, 547, 557, 536, 481, 518, 533, 491, 499, 526, 513, 479, 505, 497, 537, 484, 485, 506, 532, 483, 524, 509, 503, 496, 549, 514, 504, 550, 485, 495] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[543, 538, 533, 532, 513, 523, 522, 527, 501, 535, 497, 483, 515, 496, 489, 569, 525, 514, 521, 521, 499, 489, 523, 478, 555, 510, 512, 497, 502, 527, 535, 499, 492, 510, 512, 502, 502, 534, 517, 500, 454, 527, 528, 523, 504, 491, 491, 493, 531, 491, 539, 534, 548, 515, 504, 488, 446, 487, 534, 523, 503, 535, 515, 472] tokens_per_ep_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] avg=32768.0000 max=32768 max_rank=0 min=32768 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[513, 509, 535, 526, 524, 445, 562, 518, 0, 0, 0, 0, 0, 0, 0, 0, 502, 504, 460, 488, 518, 531, 483, 489, 0, 0, 0, 0, 0, 0, 0, 0, 477, 513, 560, 569, 566, 471, 527, 519, 0, 0, 0, 0, 0, 0, 0, 0, 521, 485, 529, 466, 502, 544, 517, 502, 6, 0, 0, 0, 0, 0, 0, 0, 517, 504, 548, 500, 537, 481, 502, 499, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 484, 500, 465, 541, 519, 512, 56, 17, 0, 0, 0, 0, 0, 0, 495, 530, 497, 508, 519, 536, 507, 533, 0, 0, 0, 0, 0, 0, 0, 0, 501, 500, 484, 521, 541, 510, 487, 494, 26, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4132, 3975, 4202, 4072, 4088, 4097, 4125, 4077] avg=4096.0000 max=4202 max_rank=2 min=3975 min_rank=1 imbalance_ratio=1.025879 max_load_reduction=28566 max_load_reduction_pct=0.871765 moved_to_echo=16387 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[510, 486, 486, 503, 485, 467, 514, 524, 0, 0, 0, 0, 0, 0, 0, 0, 488, 518, 486, 520, 530, 559, 502, 489, 0, 0, 0, 0, 0, 0, 0, 0, 524, 509, 526, 498, 527, 545, 481, 516, 0, 0, 0, 0, 0, 0, 0, 0, 523, 509, 490, 500, 500, 558, 514, 510, 5, 0, 0, 0, 0, 0, 0, 0, 520, 526, 537, 475, 493, 462, 507, 504, 0, 0, 0, 0, 0, 0, 0, 0, 519, 516, 509, 495, 522, 497, 482, 517, 59, 17, 0, 0, 0, 0, 0, 0, 552, 527, 516, 499, 490, 530, 501, 538, 0, 0, 0, 0, 0, 0, 0, 0, 519, 506, 513, 494, 545, 482, 523, 534, 27, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3975, 4092, 4126, 4109, 4024, 4133, 4153, 4156] avg=4096.0000 max=4156 max_rank=7 min=3975 min_rank=0 imbalance_ratio=1.014648 max_load_reduction=28612 max_load_reduction_pct=0.873169 moved_to_echo=16466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[520, 534, 476, 508, 483, 494, 548, 534, 0, 0, 0, 0, 0, 0, 0, 0, 549, 543, 524, 530, 530, 513, 512, 471, 0, 0, 0, 0, 0, 0, 0, 0, 498, 532, 479, 528, 501, 490, 519, 525, 0, 0, 0, 0, 0, 0, 0, 0, 514, 474, 499, 479, 497, 486, 531, 498, 6, 0, 0, 0, 0, 0, 0, 0, 528, 518, 515, 525, 562, 492, 519, 487, 0, 0, 0, 0, 0, 0, 0, 0, 514, 467, 512, 507, 479, 515, 496, 479, 63, 17, 0, 0, 0, 0, 0, 0, 522, 514, 514, 544, 526, 496, 548, 502, 0, 0, 0, 0, 0, 0, 0, 0, 528, 490, 485, 505, 528, 485, 503, 518, 27, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4097, 4172, 4072, 3984, 4146, 4049, 4166, 4082] avg=4096.0000 max=4172 max_rank=1 min=3984 min_rank=3 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16443 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[509, 514, 531, 535, 506, 469, 523, 534, 0, 0, 0, 0, 0, 0, 0, 0, 479, 518, 515, 510, 521, 497, 528, 493, 0, 0, 0, 0, 0, 0, 0, 0, 498, 517, 508, 486, 508, 492, 537, 516, 0, 0, 0, 0, 0, 0, 0, 0, 496, 499, 511, 510, 515, 492, 516, 504, 5, 0, 0, 0, 0, 0, 0, 0, 536, 495, 516, 547, 557, 536, 481, 518, 0, 0, 0, 0, 0, 0, 0, 0, 533, 491, 499, 526, 513, 479, 505, 497, 60, 18, 0, 0, 0, 0, 0, 0, 537, 484, 461, 506, 532, 483, 524, 509, 0, 0, 0, 0, 0, 0, 0, 0, 503, 496, 549, 514, 504, 550, 485, 495, 24, 13, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4121, 4061, 4062, 4048, 4186, 4121, 4036, 4133] avg=4096.0000 max=4186 max_rank=4 min=4036 min_rank=6 imbalance_ratio=1.021973 max_load_reduction=28582 max_load_reduction_pct=0.872253 moved_to_echo=16476 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[567, 507, 503, 514, 510, 475, 487, 498, 0, 0, 0, 0, 0, 0, 0, 0, 521, 518, 513, 529, 521, 485, 573, 510, 0, 0, 0, 0, 0, 0, 0, 0, 496, 530, 509, 466, 514, 495, 506, 551, 0, 0, 0, 0, 0, 0, 0, 0, 522, 536, 518, 531, 536, 544, 494, 537, 5, 0, 0, 0, 0, 0, 0, 0, 490, 474, 499, 547, 492, 496, 515, 485, 0, 0, 0, 0, 0, 0, 0, 0, 503, 522, 509, 482, 497, 555, 527, 514, 60, 18, 0, 0, 0, 0, 0, 0, 524, 509, 503, 491, 494, 508, 536, 472, 0, 0, 0, 0, 0, 0, 0, 0, 508, 526, 501, 484, 521, 472, 479, 494, 26, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4061, 4170, 4067, 4223, 3998, 4187, 4037, 4025] avg=4096.0000 max=4223 max_rank=3 min=3998 min_rank=4 imbalance_ratio=1.031006 max_load_reduction=28545 max_load_reduction_pct=0.871124 moved_to_echo=16247 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[514, 530, 476, 505, 552, 489, 525, 490, 0, 0, 0, 0, 0, 0, 0, 0, 551, 544, 533, 533, 452, 527, 511, 541, 0, 0, 0, 0, 0, 0, 0, 0, 497, 494, 508, 497, 484, 534, 498, 485, 0, 0, 0, 0, 0, 0, 0, 0, 546, 532, 508, 504, 527, 487, 507, 498, 5, 0, 0, 0, 0, 0, 0, 0, 512, 517, 535, 535, 521, 505, 521, 518, 0, 0, 0, 0, 0, 0, 0, 0, 496, 509, 518, 483, 509, 494, 514, 502, 62, 19, 0, 0, 0, 0, 0, 0, 498, 494, 507, 523, 521, 525, 479, 507, 0, 0, 0, 0, 0, 0, 0, 0, 511, 492, 510, 472, 525, 512, 524, 472, 27, 15, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4081, 4192, 3997, 4114, 4164, 4106, 4054, 4060] avg=4096.0000 max=4192 max_rank=1 min=3997 min_rank=2 imbalance_ratio=1.023438 max_load_reduction=28576 max_load_reduction_pct=0.872070 moved_to_echo=16384 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[547, 543, 521, 513, 509, 455, 556, 485, 0, 0, 0, 0, 0, 0, 0, 0, 554, 536, 483, 476, 523, 501, 512, 469, 0, 0, 0, 0, 0, 0, 0, 0, 526, 541, 505, 549, 531, 501, 506, 518, 0, 0, 0, 0, 0, 0, 0, 0, 492, 502, 548, 515, 513, 481, 516, 491, 10, 7, 1, 0, 0, 0, 0, 0, 502, 467, 518, 513, 520, 513, 539, 521, 0, 0, 0, 0, 0, 0, 0, 0, 463, 456, 509, 511, 493, 481, 500, 489, 62, 22, 0, 0, 0, 0, 0, 0, 474, 492, 476, 491, 528, 523, 549, 541, 0, 0, 0, 0, 0, 0, 0, 0, 527, 500, 490, 521, 523, 503, 522, 546, 30, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4129, 4054, 4177, 4076, 4093, 3986, 4074, 4179] avg=4096.0000 max=4179 max_rank=7 min=3986 min_rank=5 imbalance_ratio=1.020264 max_load_reduction=28589 max_load_reduction_pct=0.872467 moved_to_echo=16332 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[543, 538, 533, 532, 513, 464, 522, 527, 0, 0, 0, 0, 0, 0, 0, 0, 501, 535, 497, 483, 515, 496, 489, 536, 0, 0, 0, 0, 0, 0, 0, 0, 525, 514, 521, 516, 499, 489, 523, 478, 0, 0, 0, 0, 0, 0, 0, 0, 555, 510, 512, 497, 502, 527, 535, 499, 5, 0, 0, 0, 0, 0, 0, 0, 492, 510, 512, 502, 502, 534, 517, 500, 0, 0, 0, 0, 0, 0, 0, 0, 454, 527, 528, 523, 504, 491, 491, 493, 59, 19, 0, 0, 0, 0, 0, 0, 531, 491, 512, 534, 548, 515, 504, 488, 0, 0, 0, 0, 0, 0, 0, 0, 446, 487, 534, 523, 503, 535, 515, 472, 27, 14, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4172, 4052, 4065, 4142, 4069, 4089, 4123, 4056] avg=4096.0000 max=4172 max_rank=0 min=4052 min_rank=1 imbalance_ratio=1.018555 max_load_reduction=28596 max_load_reduction_pct=0.872681 moved_to_echo=16337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=4096 router_topk=8 budget=65536 expected_recv=32768 expected_overflow=0 exact_recv_per_rank=[32768, 32768, 32768, 32768, 32768, 32768, 32768, 32768] +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + + + +HybridEP only supports float32 probs, please set --moe-router-dtype=fp32HybridEP only supports float32 probs, please set --moe-router-dtype=fp32 + +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4224, 4096, 4224, 4096, 3840, 4352, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=65536 expected_recv=32768 actual_valid_recv=33664 observed_loss=-896 tokens_per_expert=[4224, 4096, 4224, 4096, 4096, 4224, 4224, 4096, 128, 128, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4224, 4224, 4224, 4224, 4096, 4224, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=65536 expected_recv=32768 actual_valid_recv=33536 observed_loss=-768 tokens_per_expert=[4096, 4096, 4096, 4096, 4096, 4096, 4096, 4096, 512, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4224, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4096, 4224, 4224, 4224, 4096, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=65536 expected_recv=32768 actual_valid_recv=33280 observed_loss=-512 tokens_per_expert=[4096, 4096, 4096, 4096, 4224, 4096, 4096, 4096, 256, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=65536 expected_recv=32768 actual_valid_recv=33408 observed_loss=-640 tokens_per_expert=[4224, 4224, 4096, 4096, 4224, 4224, 4224, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=65536 valid_num_tokens=33664 slack_slots=31872 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=65536 valid_num_tokens=33536 slack_slots=32000 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=65536 valid_num_tokens=33280 slack_slots=32256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=65536 valid_num_tokens=33408 slack_slots=32128 device_initiated_grouped_gemm=True + [2026-08-26 14:57:44] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 285.1 | throughput per GPU (TFLOP/s/GPU): 433.1 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.434260E+00 | loss scale: 1.0 | grad norm: 3.432 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 14:57:44 +[rank0]:[W826 14:57:45.824504079 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 14:57:47.086000 40721 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 14:57:47.087000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40812 closing signal SIGTERM +W0826 14:57:47.087000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40813 closing signal SIGTERM +W0826 14:57:47.087000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40814 closing signal SIGTERM +W0826 14:57:47.088000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40815 closing signal SIGTERM +W0826 14:57:47.088000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40817 closing signal SIGTERM +W0826 14:57:47.088000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40818 closing signal SIGTERM +W0826 14:57:47.088000 40721 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 40819 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 40721 got signal: 15 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index 970a60ec13026bad60eef8d8cf62448199b56840..cab80b3e0b505667f8e39074574c08e9812d9317 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -6,3 +6,4 @@ QWEN3_30B echo forced 2.00 OK 132963 1 0.0 1.0 /workspace/tierb_study_20260826_1 DSV2_LITE_L24 echo natural 2.00 OK 140961 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo forced 2.00 OK 140541 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 QWEN2_57B echo natural 2.00 OK 108303 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN2_57B echo forced 2.00 OK 108287 1 0.0 1.0 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index a89a8a6eccee134370cc525ac9c9042dc9e1c94b..46b5e16a387e211a42ba27c065f15e2015290151 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -2,3 +2,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib QWEN3_235B echo SELECTED 2.00 1.0 0.0 141943 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 QWEN3_30B echo SELECTED 2.00 1.0 0.0 133531 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo SELECTED 2.00 1.0 0.0 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 +QWEN2_57B echo SELECTED 2.00 1.0 0.0 108303 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..078922113f45701185826a3f8787c7f05fecdd8b --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.json @@ -0,0 +1,690 @@ +[ + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.59375, + "cold_expert_fraction_mean": 0.12777400362318841, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 276, + "commit": "b099fd84", + "config": "DSV2_LITE_L24", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 108527616.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 276, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 276, + "hottest_expert_token_fraction_max": 0.999969482421875, + "hottest_expert_token_fraction_mean": 0.6977317920629529, + "hottest_expert_token_fraction_min": 0.18292236328125, + "hottest_expert_token_fraction_n": 276, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "23", + "num_experts": "64", + "num_layers": "24", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 276, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 276, + "pre_home_rank_cv_max": 1.3955462296743573, + "pre_home_rank_cv_mean": 0.5956958266809962, + "pre_home_rank_cv_min": 0.06838324069662365, + "pre_home_rank_cv_n": 276, + "pre_home_rank_imax_max": 4.433919270833333, + "pre_home_rank_imax_mean": 2.117264752226751, + "pre_home_rank_imax_min": 1.1109822591145833, + "pre_home_rank_imax_n": 276, + "raw_expert_cv_max": 3.0650012939743223, + "raw_expert_cv_mean": 1.7377059172733105, + "raw_expert_cv_min": 0.36860618049647703, + "raw_expert_cv_n": 276, + "raw_expert_imax_ceiling": 10.666666666666666, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 10.666341145833334, + "raw_expert_imax_mean": 7.442472448671498, + "raw_expert_imax_min": 1.951171875, + "raw_expert_imax_n": 276, + "records": { + "drop_dispatch": 2208, + "drop_experts": 2208, + "drop_setup": 2208, + "echo_post": 2208, + "echo_pre": 2208 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 2208, + "router_balancing": "seq_aux_loss", + "run_dir": "DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "6", + "valid_max_rank_reduction_max": 0.7744658982453564, + "valid_max_rank_reduction_mean": 0.4729061659118927, + "valid_max_rank_reduction_min": 0.09989561777794055, + "valid_max_rank_reduction_n": 276, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 276, + "commit": "b099fd84", + "config": "DSV2_LITE_L24", + "cuda_graph": "0", + "dense_first_layers": "1", + "drop_expected_assignments": 108527616.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 276, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 276, + "hottest_expert_token_fraction_max": 0.0982818603515625, + "hottest_expert_token_fraction_mean": 0.09644090956535892, + "hottest_expert_token_fraction_min": 0.09539794921875, + "hottest_expert_token_fraction_n": 276, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "23", + "num_experts": "64", + "num_layers": "24", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 276, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 276, + "pre_home_rank_cv_max": 0.006715970721309454, + "pre_home_rank_cv_mean": 0.00392157619549587, + "pre_home_rank_cv_min": 0.0014708009315538319, + "pre_home_rank_cv_n": 276, + "pre_home_rank_imax_max": 1.0122884114583333, + "pre_home_rank_imax_mean": 1.0060781580238527, + "pre_home_rank_imax_min": 1.00213623046875, + "pre_home_rank_imax_n": 276, + "raw_expert_cv_max": 0.015123587213954862, + "raw_expert_cv_mean": 0.012112028161640214, + "raw_expert_cv_min": 0.008171875787738593, + "raw_expert_cv_n": 276, + "raw_expert_imax_ceiling": 10.666666666666666, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.04833984375, + "raw_expert_imax_mean": 1.0287030353638287, + "raw_expert_imax_min": 1.017578125, + "raw_expert_imax_n": 276, + "records": { + "drop_dispatch": 2208, + "drop_experts": 2208, + "drop_setup": 2208, + "echo_post": 2208, + "echo_pre": 2208 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 2208, + "router_balancing": "seq_aux_loss", + "run_dir": "DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "6", + "valid_max_rank_reduction_max": 0.01213923948870488, + "valid_max_rank_reduction_mean": 0.006037272492700937, + "valid_max_rank_reduction_min": 0.0021316767159997563, + "valid_max_rank_reduction_n": 276, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.828125, + "cold_expert_fraction_mean": 0.3279947916666667, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 120, + "commit": "b099fd84", + "config": "QWEN2_57B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 31457280.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 120, + "experts_per_rank": "8", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 120, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.8515007019042968, + "hottest_expert_token_fraction_min": 0.2647705078125, + "hottest_expert_token_fraction_n": 120, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "10", + "num_experts": "64", + "num_layers": "10", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 120, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 120, + "pre_home_rank_cv_max": 1.2169604837256403, + "pre_home_rank_cv_mean": 0.6377553523750428, + "pre_home_rank_cv_min": 0.09633592940981593, + "pre_home_rank_cv_n": 120, + "pre_home_rank_imax_max": 3.978790283203125, + "pre_home_rank_imax_mean": 2.1783960978190104, + "pre_home_rank_imax_min": 1.12921142578125, + "pre_home_rank_imax_n": 120, + "raw_expert_cv_max": 2.6426244377115378, + "raw_expert_cv_mean": 1.9430158591244282, + "raw_expert_cv_min": 0.5540860535283595, + "raw_expert_cv_n": 120, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 8.0, + "raw_expert_imax_mean": 6.812005615234375, + "raw_expert_imax_min": 2.1181640625, + "raw_expert_imax_n": 120, + "records": { + "drop_dispatch": 960, + "drop_experts": 960, + "drop_setup": 960, + "echo_post": 960, + "echo_pre": 960 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 960, + "router_balancing": "aux_loss", + "run_dir": "QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7486673262922141, + "valid_max_rank_reduction_mean": 0.497309315009654, + "valid_max_rank_reduction_min": 0.11442624722987947, + "valid_max_rank_reduction_n": 120, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-2", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 120, + "commit": "b099fd84", + "config": "QWEN2_57B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 31457280.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 120, + "experts_per_rank": "8", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 120, + "hottest_expert_token_fraction_max": 0.1317138671875, + "hottest_expert_token_fraction_mean": 0.129486083984375, + "hottest_expert_token_fraction_min": 0.128021240234375, + "hottest_expert_token_fraction_n": 120, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "10", + "num_experts": "64", + "num_layers": "10", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 120, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 120, + "pre_home_rank_cv_max": 0.01200287811464368, + "pre_home_rank_cv_mean": 0.0057851448972811755, + "pre_home_rank_cv_min": 0.0010965114882337452, + "pre_home_rank_cv_n": 120, + "pre_home_rank_imax_max": 1.015289306640625, + "pre_home_rank_imax_mean": 1.0091585795084634, + "pre_home_rank_imax_min": 1.001373291015625, + "pre_home_rank_imax_n": 120, + "raw_expert_cv_max": 0.01846260511411008, + "raw_expert_cv_mean": 0.01497261060068789, + "raw_expert_cv_min": 0.011926166451983852, + "raw_expert_cv_n": 120, + "raw_expert_imax_ceiling": 8.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.0537109375, + "raw_expert_imax_mean": 1.035888671875, + "raw_expert_imax_min": 1.024169921875, + "raw_expert_imax_n": 120, + "records": { + "drop_dispatch": 960, + "drop_experts": 960, + "drop_setup": 960, + "echo_post": 960, + "echo_pre": 960 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 960, + "router_balancing": "aux_loss", + "run_dir": "QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.01505906399350747, + "valid_max_rank_reduction_mean": 0.009067283410582577, + "valid_max_rank_reduction_min": 0.001371407673787828, + "valid_max_rank_reduction_n": 120, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.7890625, + "cold_expert_fraction_mean": 0.4895833333333333, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 144, + "commit": "b099fd84", + "config": "QWEN3_235B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 75497472.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 144, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 144, + "hottest_expert_token_fraction_max": 0.9999542236328125, + "hottest_expert_token_fraction_mean": 0.9661488003200955, + "hottest_expert_token_fraction_min": 0.546875, + "hottest_expert_token_fraction_n": 144, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "12", + "num_experts": "128", + "num_layers": "12", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 144, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 144, + "pre_home_rank_cv_max": 1.5534351281913334, + "pre_home_rank_cv_mean": 0.8358247018998112, + "pre_home_rank_cv_min": 0.3167748701963148, + "pre_home_rank_cv_n": 144, + "pre_home_rank_imax_max": 4.9171295166015625, + "pre_home_rank_imax_mean": 2.5417197545369468, + "pre_home_rank_imax_min": 1.2317352294921875, + "pre_home_rank_imax_n": 144, + "raw_expert_cv_max": 3.8719906493286524, + "raw_expert_cv_mean": 3.456592832253774, + "raw_expert_cv_min": 1.5070684637391225, + "raw_expert_cv_n": 144, + "raw_expert_imax_ceiling": 16.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 15.999267578125, + "raw_expert_imax_mean": 15.458380805121529, + "raw_expert_imax_min": 8.75, + "raw_expert_imax_n": 144, + "records": { + "drop_dispatch": 1152, + "drop_experts": 1152, + "drop_setup": 1152, + "echo_post": 1152, + "echo_pre": 1152 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 1152, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7966293145983385, + "valid_max_rank_reduction_mean": 0.5769520293233497, + "valid_max_rank_reduction_min": 0.18813720996494185, + "valid_max_rank_reduction_n": 144, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 144, + "commit": "b099fd84", + "config": "QWEN3_235B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 75497472.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 144, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 144, + "hottest_expert_token_fraction_max": 0.0663299560546875, + "hottest_expert_token_fraction_mean": 0.0649354722764757, + "hottest_expert_token_fraction_min": 0.0641632080078125, + "hottest_expert_token_fraction_n": 144, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "12", + "num_experts": "128", + "num_layers": "12", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 144, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 144, + "pre_home_rank_cv_max": 0.006990045273933762, + "pre_home_rank_cv_mean": 0.0034121784262079076, + "pre_home_rank_cv_min": 0.0012782366924169522, + "pre_home_rank_cv_n": 144, + "pre_home_rank_imax_max": 1.0104522705078125, + "pre_home_rank_imax_mean": 1.0053127076890733, + "pre_home_rank_imax_min": 1.0019378662109375, + "pre_home_rank_imax_n": 144, + "raw_expert_cv_max": 0.017577383237659006, + "raw_expert_cv_mean": 0.015149663454502107, + "raw_expert_cv_min": 0.013421143735031664, + "raw_expert_cv_n": 144, + "raw_expert_imax_ceiling": 16.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.061279296875, + "raw_expert_imax_mean": 1.0389675564236112, + "raw_expert_imax_min": 1.026611328125, + "raw_expert_imax_n": 144, + "records": { + "drop_dispatch": 1152, + "drop_experts": 1152, + "drop_setup": 1152, + "echo_post": 1152, + "echo_pre": 1152 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 1152, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1", + "seq_length": "8192", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.010344150647075702, + "valid_max_rank_reduction_mean": 0.005281131647695585, + "valid_max_rank_reduction_min": 0.0019341181487291168, + "valid_max_rank_reduction_n": 144, + "zero_drop": 1 + }, + { + "arm": "natural", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.9296875, + "cold_expert_fraction_mean": 0.6222466362847222, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 576, + "commit": "b099fd84", + "config": "QWEN3_30B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 150994944.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 576, + "experts_per_rank": "16", + "force_balance": "0", + "global_batch_size": "8", + "global_snapshots": 576, + "hottest_expert_token_fraction_max": 1.0, + "hottest_expert_token_fraction_mean": 0.9104721811082628, + "hottest_expert_token_fraction_min": 0.17547607421875, + "hottest_expert_token_fraction_n": 576, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "48", + "num_experts": "128", + "num_layers": "48", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 576, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 576, + "pre_home_rank_cv_max": 1.5704882864247645, + "pre_home_rank_cv_mean": 0.7589291473184087, + "pre_home_rank_cv_min": 0.13794675824562003, + "pre_home_rank_cv_n": 576, + "pre_home_rank_imax_max": 4.974578857421875, + "pre_home_rank_imax_mean": 2.3463818762037487, + "pre_home_rank_imax_min": 1.2030029296875, + "pre_home_rank_imax_n": 576, + "raw_expert_cv_max": 3.8726526007751483, + "raw_expert_cv_mean": 3.3062029242663797, + "raw_expert_cv_min": 0.540439815722741, + "raw_expert_cv_n": 576, + "raw_expert_imax_ceiling": 16.0, + "raw_expert_imax_hit_ceiling": 1, + "raw_expert_imax_max": 16.0, + "raw_expert_imax_mean": 14.567554897732204, + "raw_expert_imax_min": 2.8076171875, + "raw_expert_imax_n": 576, + "records": { + "drop_dispatch": 4608, + "drop_experts": 4608, + "drop_setup": 4608, + "echo_post": 4608, + "echo_pre": 4608 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 4608, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.7989779580018036, + "valid_max_rank_reduction_mean": 0.5410548593820177, + "valid_max_rank_reduction_min": 0.16874682902080163, + "valid_max_rank_reduction_n": 576, + "zero_drop": 1 + }, + { + "arm": "forced", + "assignment_sum_mismatches": 0, + "aux_loss_coeff": "1e-3", + "backend": "echo", + "branch": "token-drop-analysis", + "capacity": "2.00", + "capacity_mode": "static", + "cold_expert_fraction_max": 0.0, + "cold_expert_fraction_mean": 0.0, + "cold_expert_fraction_min": 0.0, + "cold_expert_fraction_n": 576, + "commit": "b099fd84", + "config": "QWEN3_30B", + "cuda_graph": "0", + "dense_first_layers": "0", + "drop_expected_assignments": 150994944.0, + "drop_rate_pct_overall": 0.0, + "echo_logged_load_reduction_valid": 0, + "echo_logged_pre_rank_metric_valid": 0, + "echo_moved_assignment_fraction_max": 0.5, + "echo_moved_assignment_fraction_mean": 0.5, + "echo_moved_assignment_fraction_min": 0.5, + "echo_moved_assignment_fraction_n": 576, + "experts_per_rank": "16", + "force_balance": "1", + "global_batch_size": "8", + "global_snapshots": 576, + "hottest_expert_token_fraction_max": 0.06854248046875, + "hottest_expert_token_fraction_mean": 0.066029601626926, + "hottest_expert_token_fraction_min": 0.06488037109375, + "hottest_expert_token_fraction_n": 576, + "incomplete_snapshots": 0, + "malformed_records": 0, + "moe_layers": "48", + "num_experts": "128", + "num_layers": "48", + "observed_dropped_tokens": 0.0, + "post_physical_rank_cv_max": 0.0, + "post_physical_rank_cv_mean": 0.0, + "post_physical_rank_cv_min": 0.0, + "post_physical_rank_cv_n": 576, + "post_physical_rank_imax_max": 1.0, + "post_physical_rank_imax_mean": 1.0, + "post_physical_rank_imax_min": 1.0, + "post_physical_rank_imax_n": 576, + "pre_home_rank_cv_max": 0.009112333689945906, + "pre_home_rank_cv_mean": 0.004932308997139817, + "pre_home_rank_cv_min": 0.0012089160228135533, + "pre_home_rank_cv_n": 576, + "pre_home_rank_imax_max": 1.01739501953125, + "pre_home_rank_imax_mean": 1.0077196756998699, + "pre_home_rank_imax_min": 1.00177001953125, + "pre_home_rank_imax_n": 576, + "raw_expert_cv_max": 0.02488528396573193, + "raw_expert_cv_mean": 0.021371222454350164, + "raw_expert_cv_min": 0.018128366157757106, + "raw_expert_cv_n": 576, + "raw_expert_imax_ceiling": 16.0, + "raw_expert_imax_hit_ceiling": 0, + "raw_expert_imax_max": 1.0966796875, + "raw_expert_imax_mean": 1.056473626030816, + "raw_expert_imax_min": 1.0380859375, + "raw_expert_imax_n": 576, + "records": { + "drop_dispatch": 4608, + "drop_experts": 4608, + "drop_setup": 4608, + "echo_post": 4608, + "echo_pre": 4608 + }, + "required_capacity_max": 1.0, + "required_capacity_mean": 1.0, + "required_capacity_min": 1.0, + "required_capacity_n": 4608, + "router_balancing": "aux_loss", + "run_dir": "QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1", + "seq_length": "4096", + "shared_expert_overlap": "0", + "topk": "8", + "valid_max_rank_reduction_max": 0.017097606335113084, + "valid_max_rank_reduction_mean": 0.00765283599899436, + "valid_max_rank_reduction_min": 0.0017668920977274112, + "valid_max_rank_reduction_n": 576, + "zero_drop": 1 + } +] \ No newline at end of file diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv new file mode 100644 index 0000000000000000000000000000000000000000..9365b210c63c49305a90cdd021aa0b9fa4b133fb --- /dev/null +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/probe_summary.tsv @@ -0,0 +1,9 @@ +run_dir arm config backend num_experts topk experts_per_rank num_layers moe_layers dense_first_layers seq_length global_batch_size capacity_mode capacity force_balance cuda_graph branch commit router_balancing aux_loss_coeff shared_expert_overlap assignment_sum_mismatches cold_expert_fraction_max cold_expert_fraction_mean cold_expert_fraction_min cold_expert_fraction_n drop_expected_assignments drop_rate_pct_overall echo_logged_load_reduction_valid echo_logged_pre_rank_metric_valid echo_moved_assignment_fraction_max echo_moved_assignment_fraction_mean echo_moved_assignment_fraction_min echo_moved_assignment_fraction_n global_snapshots hottest_expert_token_fraction_max hottest_expert_token_fraction_mean hottest_expert_token_fraction_min hottest_expert_token_fraction_n incomplete_snapshots malformed_records observed_dropped_tokens post_physical_rank_cv_max post_physical_rank_cv_mean post_physical_rank_cv_min post_physical_rank_cv_n post_physical_rank_imax_max post_physical_rank_imax_mean post_physical_rank_imax_min post_physical_rank_imax_n pre_home_rank_cv_max pre_home_rank_cv_mean pre_home_rank_cv_min pre_home_rank_cv_n pre_home_rank_imax_max pre_home_rank_imax_mean pre_home_rank_imax_min pre_home_rank_imax_n raw_expert_cv_max raw_expert_cv_mean raw_expert_cv_min raw_expert_cv_n raw_expert_imax_ceiling raw_expert_imax_hit_ceiling raw_expert_imax_max raw_expert_imax_mean raw_expert_imax_min raw_expert_imax_n required_capacity_max required_capacity_mean required_capacity_min required_capacity_n valid_max_rank_reduction_max valid_max_rank_reduction_mean valid_max_rank_reduction_min valid_max_rank_reduction_n zero_drop +DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 natural DSV2_LITE_L24 echo 64 6 8 24 23 1 8192 8 static 2.00 0 0 token-drop-analysis b099fd84 seq_aux_loss 1e-3 0 0 0.59375 0.12777400362318841 0.0 276 108527616.0 0.0 0 0 0.5 0.5 0.5 276 276 0.999969482421875 0.6977317920629529 0.18292236328125 276 0 0 0.0 0.0 0.0 0.0 276 1.0 1.0 1.0 276 1.3955462296743573 0.5956958266809962 0.06838324069662365 276 4.433919270833333 2.117264752226751 1.1109822591145833 276 3.0650012939743223 1.7377059172733105 0.36860618049647703 276 10.666666666666666 1 10.666341145833334 7.442472448671498 1.951171875 276 1.0 1.0 1.0 2208 0.7744658982453564 0.4729061659118927 0.09989561777794055 276 1 +DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 forced DSV2_LITE_L24 echo 64 6 8 24 23 1 8192 8 static 2.00 1 0 token-drop-analysis b099fd84 seq_aux_loss 1e-3 0 0 0.0 0.0 0.0 276 108527616.0 0.0 0 0 0.5 0.5 0.5 276 276 0.0982818603515625 0.09644090956535892 0.09539794921875 276 0 0 0.0 0.0 0.0 0.0 276 1.0 1.0 1.0 276 0.006715970721309454 0.00392157619549587 0.0014708009315538319 276 1.0122884114583333 1.0060781580238527 1.00213623046875 276 0.015123587213954862 0.012112028161640214 0.008171875787738593 276 10.666666666666666 0 1.04833984375 1.0287030353638287 1.017578125 276 1.0 1.0 1.0 2208 0.01213923948870488 0.006037272492700937 0.0021316767159997563 276 1 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.828125 0.3279947916666667 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 1.0 0.8515007019042968 0.2647705078125 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 1.2169604837256403 0.6377553523750428 0.09633592940981593 120 3.978790283203125 2.1783960978190104 1.12921142578125 120 2.6426244377115378 1.9430158591244282 0.5540860535283595 120 8.0 1 8.0 6.812005615234375 2.1181640625 120 1.0 1.0 1.0 960 0.7486673262922141 0.497309315009654 0.11442624722987947 120 1 +QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 forced QWEN2_57B echo 64 8 8 10 10 0 4096 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-2 0 0 0.0 0.0 0.0 120 31457280.0 0.0 0 0 0.5 0.5 0.5 120 120 0.1317138671875 0.129486083984375 0.128021240234375 120 0 0 0.0 0.0 0.0 0.0 120 1.0 1.0 1.0 120 0.01200287811464368 0.0057851448972811755 0.0010965114882337452 120 1.015289306640625 1.0091585795084634 1.001373291015625 120 0.01846260511411008 0.01497261060068789 0.011926166451983852 120 8.0 0 1.0537109375 1.035888671875 1.024169921875 120 1.0 1.0 1.0 960 0.01505906399350747 0.009067283410582577 0.001371407673787828 120 1 +QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00/rep1 natural QWEN3_235B echo 128 8 16 12 12 0 8192 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.7890625 0.4895833333333333 0.0 144 75497472.0 0.0 0 0 0.5 0.5 0.5 144 144 0.9999542236328125 0.9661488003200955 0.546875 144 0 0 0.0 0.0 0.0 0.0 144 1.0 1.0 1.0 144 1.5534351281913334 0.8358247018998112 0.3167748701963148 144 4.9171295166015625 2.5417197545369468 1.2317352294921875 144 3.8719906493286524 3.456592832253774 1.5070684637391225 144 16.0 1 15.999267578125 15.458380805121529 8.75 144 1.0 1.0 1.0 1152 0.7966293145983385 0.5769520293233497 0.18813720996494185 144 1 +QWEN3_235B__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 forced QWEN3_235B echo 128 8 16 12 12 0 8192 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.0 0.0 0.0 144 75497472.0 0.0 0 0 0.5 0.5 0.5 144 144 0.0663299560546875 0.0649354722764757 0.0641632080078125 144 0 0 0.0 0.0 0.0 0.0 144 1.0 1.0 1.0 144 0.006990045273933762 0.0034121784262079076 0.0012782366924169522 144 1.0104522705078125 1.0053127076890733 1.0019378662109375 144 0.017577383237659006 0.015149663454502107 0.013421143735031664 144 16.0 0 1.061279296875 1.0389675564236112 1.026611328125 144 1.0 1.0 1.0 1152 0.010344150647075702 0.005281131647695585 0.0019341181487291168 144 1 +QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00/rep1 natural QWEN3_30B echo 128 8 16 48 48 0 4096 8 static 2.00 0 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.9296875 0.6222466362847222 0.0 576 150994944.0 0.0 0 0 0.5 0.5 0.5 576 576 1.0 0.9104721811082628 0.17547607421875 576 0 0 0.0 0.0 0.0 0.0 576 1.0 1.0 1.0 576 1.5704882864247645 0.7589291473184087 0.13794675824562003 576 4.974578857421875 2.3463818762037487 1.2030029296875 576 3.8726526007751483 3.3062029242663797 0.540439815722741 576 16.0 1 16.0 14.567554897732204 2.8076171875 576 1.0 1.0 1.0 4608 0.7989779580018036 0.5410548593820177 0.16874682902080163 576 1 +QWEN3_30B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1 forced QWEN3_30B echo 128 8 16 48 48 0 4096 8 static 2.00 1 0 token-drop-analysis b099fd84 aux_loss 1e-3 0 0 0.0 0.0 0.0 576 150994944.0 0.0 0 0 0.5 0.5 0.5 576 576 0.06854248046875 0.066029601626926 0.06488037109375 576 0 0 0.0 0.0 0.0 0.0 576 1.0 1.0 1.0 576 0.009112333689945906 0.004932308997139817 0.0012089160228135533 576 1.01739501953125 1.0077196756998699 1.00177001953125 576 0.02488528396573193 0.021371222454350164 0.018128366157757106 576 16.0 0 1.0966796875 1.056473626030816 1.0380859375 576 1.0 1.0 1.0 4608 0.017097606335113084 0.00765283599899436 0.0017668920977274112 576 1 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv index 4bda765dc2e9e488d5759e93345d5812cce6b974..fc867ced8c824d45d248c4efaf53575df0629615 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/results.tsv @@ -6,3 +6,4 @@ QWEN3_30B echo matched 16 4096 8 2.00 none+forcebal 1 OK 954.76 910.80 132963 /w DSV2_LITE_L24 echo matched 8 8192 8 2.00 none 1 OK 826.93 818.00 140961 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00/rep1 DSV2_LITE_L24 echo matched 8 8192 8 2.00 none+forcebal 1 OK 817.73 813.10 140541 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap2p00__forcebal/rep1 QWEN2_57B echo matched 8 4096 8 2.00 none 1 OK 281.00 277.80 108303 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00/rep1 +QWEN2_57B echo matched 8 4096 8 2.00 none+forcebal 1 OK 281.80 277.00 108287 /workspace/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/QWEN2_57B__echo__matched__seq4096__gbs8__cap2p00__forcebal/rep1