diff --git a/tierb_study_20260826_144644/STATUS.txt b/tierb_study_20260826_144644/STATUS.txt index 68a653f33dc74c616f74cc05394d1ef4ecb07616..9e64996a222e1c0dc11c01dad1441b7b6b47561b 100644 --- a/tierb_study_20260826_144644/STATUS.txt +++ b/tierb_study_20260826_144644/STATUS.txt @@ -1,6 +1,6 @@ study_root: /workspace/tierb_study_20260826_144644 gbs: 8 -updated: 2026-08-26T17:09:11+00:00 +updated: 2026-08-26T17:21:48+00:00 current: echo__tierbprobe__gbs8 completed steps: diff --git a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt index 85d9951f9271098867f7f6f74beb6d36b794abec..476e384805a101c7ddb0ca9d1fd3c879874ae45f 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt +++ b/tierb_study_20260826_144644/echo/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: echo__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T17:09:11+00:00 +captured_at : 2026-08-26T17:21:48+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -22,69 +22,8 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 2, 120 MHz, 1965 MHz, 1000.00 W, 32 3, 120 MHz, 1965 MHz, 1000.00 W, 29 4, 120 MHz, 1965 MHz, 1000.00 W, 28 - 5, 120 MHz, 1965 MHz, 1000.00 W, 33 + 5, 120 MHz, 1965 MHz, 1000.00 W, 32 6, 120 MHz, 1965 MHz, 1000.00 W, 32 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: - GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID - GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A - GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A - GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A - GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A - GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A - GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A - GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A - GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A - NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE - NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE - NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS - NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS - NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS - unavailable - ----- toolchain ----------------------------------------------- - nvcc : release 12.9, V12.9.86 - gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 - python : 3.12.3 - torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 - torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] - transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] - deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] - ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] - moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] - megatron : 0.14.0rc5 [None] - flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] - numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] - huggingface_hub: 1.28.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] - ----- repositories -------------------------------------------- - Megatron-MoE-Echo token-drop-analysis b099fd8494b0 - Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 - UltraEP main 94cab099b44f - DeepEP HEAD 34152ae28f80 (+3 untracked: build output) - MoonEP megatron-integration c27d35da15f2 - blackwell_run_scripts granularity-echo 655301106287 - ----- environment --------------------------------------------- - CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 - CUDA_DRIVER_VERSION=575.57.08 - CUDA_HOME=/usr/local/cuda - CUDA_MODULE_LOADING=LAZY - CUDA_VERSION=12.9.1.010 - HF_REMOTE_ROOT=tierb_study_20260826_144644 - NCCL_VERSION=2.27.3 - NVSHMEM_VERSION=3.2.5 - PYTORCH_BUILD_NUMBER=0 - PYTORCH_BUILD_VERSION=2.8.0a0+5228986 - PYTORCH_HOME=/opt/pytorch/pytorch - PYTORCH_VERSION=2.8.0a0+5228986 - TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 - TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX - TORCH_NCCL_USE_COMM_NONBLOCKING=0 - ----- shared library resolution ------------------------------- - libnvshmem_host.so: not in ldconfig - libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 - lrwxrwxrwx 1 root root 20 Aug 26 14:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 - -rw-r--r-- 1 root root 179045744 Aug 26 14:09 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv index c51deeeaeb552119e9b5ea72c1a8c50621f5a287..054285937bdd53acacf3b305fa69b627d15792e9 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/mem_samples.csv @@ -38,3 +38,107 @@ 5, 4 6, 4 7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 27 +7, 4 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 +0, 1955 +1, 1955 +2, 1955 +3, 1955 +4, 1955 +5, 1955 +6, 1955 +7, 1955 +0, 3535 +1, 3789 +2, 3583 +3, 3547 +4, 3789 +5, 3767 +6, 3667 +7, 3727 +0, 26591 +1, 26663 +2, 26543 +3, 26529 +4, 26537 +5, 26543 +6, 26637 +7, 26701 +0, 36387 +1, 36481 +2, 36341 +3, 36329 +4, 36311 +5, 36299 +6, 36451 +7, 36455 +0, 37783 +1, 37867 +2, 37717 +3, 37715 +4, 37715 +5, 37713 +6, 37849 +7, 37867 +0, 37871 +1, 37955 +2, 37805 +3, 37803 +4, 37803 +5, 37801 +6, 37937 +7, 37955 +0, 37873 +1, 37957 +2, 37807 +3, 37805 +4, 37805 +5, 37803 +6, 37939 +7, 37957 +0, 38819 +1, 38903 +2, 38753 +3, 38753 +4, 38753 +5, 38749 +6, 38885 +7, 38903 +0, 38819 +1, 38903 +2, 38753 +3, 38753 +4, 38753 +5, 38749 +6, 38885 +7, 38903 +0, 38825 +1, 38909 +2, 38759 +3, 38759 +4, 38759 +5, 38755 +6, 38891 +7, 38909 +0, 38827 +1, 38913 +2, 38761 +3, 38761 +4, 38761 +5, 38759 +6, 38893 +7, 38913 diff --git a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log index eb79825ab07b82459b06f8ceef275b9c0831c12c..52ddc9f491c4af934a41325868f8741a86ce2519 100644 --- a/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log +++ b/tierb_study_20260826_144644/echo/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375/rep1/train.log @@ -11,3 +11,2450 @@ W0826 17:19:06.243000 350624 torch/distributed/run.py:766] W0826 17:19:06.243000 350624 torch/distributed/run.py:766] ***************************************** W0826 17:19:06.243000 350624 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. W0826 17:19:06.243000 350624 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... False + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 24 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 10944 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... False + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 2048 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 512 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_debug_drop_logging .......................... True + moe_debug_drop_logging_interval ................. 1 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ True + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. True + moe_enable_hybridep ............................. True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1408 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. [0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1] + moe_layer_recompute ............................. False + moe_num_echo_experts ............................ 64 + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... 1.0234375 + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. seq_aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... True + moe_router_score_function ....................... softmax + moe_router_topk ................................. 6 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. 2816 + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 0.707 + mscale_all_dim .................................. 0.707 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... True + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 16 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 64 + num_layers ...................................... 24 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 1 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 10000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 40.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +[Gloo] Rank [Gloo] Rank 3 is connected to 07 is connected to [Gloo] Rank peer ranks. 7[Gloo] Rank 1Expected number of connected peer ranks is : peer ranks. 2 is connected to 7Expected number of connected peer ranks is : is connected to 7 +77 peer ranks. [Gloo] Rank +[Gloo] Rank peer ranks. Expected number of connected peer ranks is : 6Expected number of connected peer ranks is : 47 is connected to 7 is connected to +7 +7 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank [Gloo] Rank 75 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 54 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 +[rank7]:[W826 17:19:24.307826677 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 17:19:24.311465105 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 17:19:24.312953319 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 17:19:24.313409161 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 17:19:24.313552398 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 17:19:24.317965791 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 17:19:24.319721114 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.035 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W826 17:19:24.504981863 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.545 seconds +time to initialize megatron (seconds): 7.130 +[after megatron is initialized] datetime: 2026-08-26 17:19:29 +building GPT model ... +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1454: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 2596386816 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 20 +Params for bucket 1 (103022592 elements, 103022592 padded size): + module.output_layer.weight +Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.weight + module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.23.self_attention.linear_kv_down_proj.weight + module.decoder.layers.23.self_attention.linear_q_proj.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight +Params for bucket 3 (40901632 elements, 40901632 padded size): + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.22.self_attention.linear_proj.weight + module.decoder.layers.22.input_layernorm.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight +Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.weight + module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight +Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_down_proj.weight + module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight +Params for bucket 6 (40901632 elements, 40901632 padded size): + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.18.input_layernorm.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.weight + module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.18.self_attention.linear_proj.weight + module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.pre_mlp_layernorm.weight +Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_kv_down_proj.weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.input_layernorm.weight +Params for bucket 8 (42734080 elements, 42734080 padded size): + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_q_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight +Params for bucket 9 (40901632 elements, 40901632 padded size): + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.14.self_attention.linear_q_proj.weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.14.input_layernorm.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.weight + module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight +Params for bucket 10 (41163264 elements, 41163264 padded size): + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight + module.decoder.layers.12.self_attention.linear_q_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.input_layernorm.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.input_layernorm.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.weight + module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight +Params for bucket 11 (42734080 elements, 42734080 padded size): + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_q_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.11.input_layernorm.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight +Params for bucket 12 (40901632 elements, 40901632 padded size): + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.10.input_layernorm.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.weight + module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.linear_q_proj.weight + module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight +Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight + module.decoder.layers.8.input_layernorm.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_q_proj.weight +Params for bucket 14 (42734080 elements, 42734080 padded size): + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_q_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.input_layernorm.weight + module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight +Params for bucket 15 (40901632 elements, 40901632 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight + module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.input_layernorm.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight +Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.4.self_attention.linear_kv_down_proj.weight + module.decoder.layers.4.self_attention.linear_q_proj.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.input_layernorm.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight +Params for bucket 17 (42734080 elements, 42734080 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_proj.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.3.self_attention.linear_kv_down_proj.weight + module.decoder.layers.3.self_attention.linear_q_proj.weight + module.decoder.layers.3.input_layernorm.weight + module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight +Params for bucket 18 (40901632 elements, 40901632 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.weight + module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight +Params for bucket 19 (71436288 elements, 71436288 padded size): + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc1.weight +Params for bucket 20 (116789760 elements, 116789760 padded size): + module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_q_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.input_layernorm.weight + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 +Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight7 + module.decoder.layers.23.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc2.weight3 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 +Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 +Params for bucket 3 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight4 + module.decoder.layers.22.mlp.experts.linear_fc1.weight3 +Params for bucket 4 (40370176 elements, 40370176 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc2.weight3 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 + module.decoder.layers.21.mlp.experts.linear_fc2.weight0 +Params for bucket 5 (40370176 elements, 40370176 padded size): + module.decoder.layers.21.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight2 + module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 +Params for bucket 6 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 +Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight7 +Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 +Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 +Params for bucket 10 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 + module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 +Params for bucket 11 (40370176 elements, 40370176 padded size): + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 +Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 +Params for bucket 13 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight7 + module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 +Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 +Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight4 + module.decoder.layers.15.mlp.experts.linear_fc1.weight3 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 +Params for bucket 16 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight5 + module.decoder.layers.14.mlp.experts.linear_fc2.weight3 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 +Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight5 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 +Params for bucket 18 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 + module.decoder.layers.13.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight3 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 +Params for bucket 19 (40370176 elements, 40370176 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 +Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight0 + module.decoder.layers.12.mlp.experts.linear_fc1.weight6 + module.decoder.layers.12.mlp.experts.linear_fc1.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 +Params for bucket 21 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 +Params for bucket 22 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 +Params for bucket 23 (40370176 elements, 40370176 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 24 (40370176 elements, 40370176 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 +Params for bucket 25 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 +Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 +Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 +Params for bucket 28 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 +Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 +Params for bucket 30 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 32 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 33 (40370176 elements, 40370176 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 +Params for bucket 34 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 +Params for bucket 35 (40370176 elements, 40370176 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 36 (40370176 elements, 40370176 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 +Params for bucket 37 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 +Params for bucket 38 (40370176 elements, 40370176 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 39 (40370176 elements, 40370176 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 +Params for bucket 40 (17301504 elements, 17301504 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 17:19:30 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-26 17:19:30 +done with setup ... +training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (671.45, 684.06) + train/valid/test-data-iterators-setup ..........: (62.54, 67.95)Overwriting rerun_state_machine.current_iteration from -1 to 0... + +[before the start of training step] datetime: 2026-08-26 17:19:30 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=2 tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 1478, 595, 527, 1414, 847, 607, 240, 1699, 627, 976, 686, 621, 761, 779, 499, 443, 934, 901, 760, 752, 519, 978, 928, 788, 889, 1055, 501, 903, 648, 599, 486, 341, 1025, 1248, 767, 658, 451, 591, 1401, 897, 677, 373, 882, 517, 662, 936, 1459, 801, 356, 633, 721, 532, 1234, 868, 625, 359, 827, 474] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=2 tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 1462, 546, 490, 1225, 1037, 548, 259, 1619, 553, 1003, 672, 646, 672, 758, 488, 461, 940, 820, 772, 849, 548, 922, 847, 702, 926, 1006, 742, 755, 563, 591, 477, 373, 911, 1248, 717, 692, 544, 485, 1420, 882, 707, 455, 794, 527, 786, 867, 1430, 1018, 314, 669, 750, 450, 1336, 918, 738, 446, 750, 464] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=2 tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 1439, 540, 551, 1080, 1080, 439, 272, 1690, 666, 1058, 619, 646, 736, 804, 541, 471, 991, 820, 721, 795, 515, 973, 860, 658, 906, 990, 484, 663, 690, 629, 554, 401, 948, 1234, 639, 695, 447, 537, 1381, 959, 688, 433, 726, 571, 951, 932, 1468, 883, 324, 662, 696, 540, 1335, 941, 616, 440, 805, 394] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=2 tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 1460, 549, 543, 1122, 956, 616, 234, 1644, 711, 1065, 634, 668, 714, 732, 547, 398, 915, 857, 703, 1209, 525, 963, 1003, 688, 811, 1045, 474, 741, 608, 628, 502, 426, 971, 1228, 792, 720, 513, 499, 1437, 837, 690, 403, 705, 538, 1037, 852, 997, 925, 344, 598, 687, 518, 1237, 991, 701, 369, 742, 426] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=2 tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 1425, 551, 493, 1125, 847, 618, 233, 1609, 616, 1132, 762, 670, 669, 798, 511, 444, 1000, 765, 684, 990, 497, 949, 788, 750, 780, 1019, 499, 640, 538, 572, 447, 407, 968, 1245, 693, 755, 504, 528, 1388, 964, 869, 440, 760, 548, 821, 911, 1227, 906, 363, 573, 687, 588, 1239, 1058, 648, 389, 763, 523] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=2 tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 1506, 614, 593, 1015, 1041, 649, 226, 1745, 575, 996, 666, 617, 772, 759, 572, 392, 1031, 930, 644, 838, 537, 1019, 882, 603, 978, 1053, 622, 917, 658, 586, 539, 343, 837, 1149, 698, 686, 443, 661, 1395, 896, 570, 452, 704, 557, 949, 753, 1397, 828, 329, 609, 736, 559, 1295, 955, 603, 346, 747, 330] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=2 tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 1422, 531, 447, 1331, 1035, 557, 255, 1588, 698, 1124, 613, 744, 668, 696, 515, 444, 1012, 839, 732, 896, 495, 1026, 865, 625, 921, 953, 625, 553, 603, 562, 485, 408, 1035, 1137, 696, 665, 505, 606, 1408, 884, 694, 459, 682, 511, 959, 848, 1512, 896, 351, 679, 662, 542, 1304, 850, 619, 405, 754, 318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=2 tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 1438, 549, 566, 1102, 864, 620, 276, 1613, 630, 990, 697, 655, 793, 721, 542, 452, 1006, 868, 744, 1048, 528, 1083, 906, 771, 907, 1038, 539, 556, 671, 610, 471, 390, 1052, 1227, 714, 723, 458, 681, 1424, 853, 682, 457, 838, 495, 879, 891, 1317, 797, 367, 638, 748, 533, 1163, 909, 714, 346, 765, 335] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1008, 980, 480, 787, 1021, 627, 890, 531, 0, 0, 0, 0, 0, 0, 0, 0, 447, 1331, 1035, 557, 255, 933, 698, 1124, 0, 0, 0, 0, 0, 0, 0, 0, 613, 744, 668, 696, 515, 444, 1012, 839, 436, 94, 0, 0, 0, 0, 0, 0, 732, 896, 495, 1026, 865, 625, 921, 487, 0, 0, 0, 0, 0, 0, 0, 0, 625, 553, 603, 562, 485, 408, 1035, 1137, 561, 0, 0, 0, 0, 0, 0, 0, 696, 665, 505, 606, 1408, 884, 694, 459, 117, 126, 0, 0, 0, 0, 0, 0, 682, 511, 959, 848, 1386, 896, 351, 679, 0, 0, 0, 0, 0, 0, 0, 0, 662, 542, 1304, 850, 619, 405, 754, 318, 349, 96, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6324, 6380, 6061, 6047, 5969, 6160, 6312, 5899] avg=6144.0000 max=6380 max_rank=1 min=5899 min_rank=7 imbalance_ratio=1.038411 max_load_reduction=42772 max_load_reduction_pct=0.870199 moved_to_echo=24340 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1110, 698, 519, 864, 620, 586, 925, 595, 0, 0, 0, 0, 0, 0, 0, 0, 527, 1414, 847, 607, 240, 998, 627, 976, 0, 0, 0, 0, 0, 0, 0, 0, 686, 621, 761, 779, 499, 443, 934, 901, 453, 101, 0, 0, 0, 0, 0, 0, 760, 752, 519, 978, 928, 788, 889, 538, 0, 0, 0, 0, 0, 0, 0, 0, 501, 903, 648, 599, 486, 341, 1025, 1248, 600, 0, 0, 0, 0, 0, 0, 0, 767, 658, 451, 591, 1401, 897, 677, 373, 130, 122, 0, 0, 0, 0, 0, 0, 882, 517, 662, 936, 1337, 801, 356, 633, 0, 0, 0, 0, 0, 0, 0, 0, 721, 532, 1234, 868, 625, 359, 827, 474, 387, 100, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5917, 6236, 6178, 6152, 6351, 6067, 6124, 6127] avg=6144.0000 max=6351 max_rank=4 min=5917 min_rank=0 imbalance_ratio=1.033691 max_load_reduction=42801 max_load_reduction_pct=0.870789 moved_to_echo=24669 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1086, 878, 503, 728, 694, 613, 899, 549, 0, 0, 0, 0, 0, 0, 0, 0, 566, 1102, 864, 620, 276, 948, 630, 990, 0, 0, 0, 0, 0, 0, 0, 0, 697, 655, 793, 721, 542, 452, 1006, 868, 441, 95, 0, 0, 0, 0, 0, 0, 744, 1048, 528, 1083, 906, 771, 907, 529, 0, 0, 0, 0, 0, 0, 0, 0, 539, 556, 671, 610, 471, 390, 1052, 1227, 570, 0, 0, 0, 0, 0, 0, 0, 714, 723, 458, 681, 1424, 853, 682, 457, 128, 110, 0, 0, 0, 0, 0, 0, 838, 495, 879, 891, 1207, 797, 367, 638, 0, 0, 0, 0, 0, 0, 0, 0, 748, 533, 1163, 909, 714, 346, 765, 335, 381, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5950, 5996, 6270, 6516, 6086, 6230, 6112, 5992] avg=6144.0000 max=6516 max_rank=3 min=5950 min_rank=0 imbalance_ratio=1.060547 max_load_reduction=42636 max_load_reduction_pct=0.867432 moved_to_echo=24420 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1136, 806, 598, 801, 1057, 568, 884, 551, 0, 0, 0, 0, 0, 0, 0, 0, 493, 1125, 847, 618, 233, 937, 616, 1132, 0, 0, 0, 0, 0, 0, 0, 0, 762, 670, 669, 798, 511, 444, 1000, 765, 440, 100, 0, 0, 0, 0, 0, 0, 684, 990, 497, 949, 788, 750, 780, 512, 0, 0, 0, 0, 0, 0, 0, 0, 499, 640, 538, 572, 447, 407, 968, 1245, 572, 0, 0, 0, 0, 0, 0, 0, 693, 755, 504, 528, 1388, 964, 869, 440, 129, 106, 0, 0, 0, 0, 0, 0, 760, 548, 821, 911, 1121, 906, 363, 573, 0, 0, 0, 0, 0, 0, 0, 0, 687, 588, 1239, 1058, 648, 389, 763, 523, 378, 101, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6401, 6001, 6159, 5950, 5888, 6376, 6003, 6374] avg=6144.0000 max=6401 max_rank=0 min=5888 min_rank=4 imbalance_ratio=1.041829 max_load_reduction=42751 max_load_reduction_pct=0.869771 moved_to_echo=24641 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1124, 897, 526, 734, 821, 523, 900, 540, 0, 0, 0, 0, 0, 0, 0, 0, 551, 1080, 1080, 439, 272, 993, 666, 1058, 0, 0, 0, 0, 0, 0, 0, 0, 619, 646, 736, 804, 541, 471, 991, 820, 441, 100, 0, 0, 0, 0, 0, 0, 721, 795, 515, 973, 860, 658, 906, 505, 0, 0, 0, 0, 0, 0, 0, 0, 484, 663, 690, 629, 554, 401, 948, 1234, 597, 0, 0, 0, 0, 0, 0, 0, 639, 695, 447, 537, 1381, 959, 688, 433, 122, 123, 0, 0, 0, 0, 0, 0, 726, 571, 951, 932, 1345, 883, 324, 662, 0, 0, 0, 0, 0, 0, 0, 0, 696, 540, 1335, 941, 616, 440, 805, 394, 363, 98, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6065, 6139, 6169, 5933, 6200, 6024, 6394, 6228] avg=6144.0000 max=6394 max_rank=6 min=5933 min_rank=3 imbalance_ratio=1.040690 max_load_reduction=42758 max_load_reduction_pct=0.869914 moved_to_echo=24846 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1069, 744, 547, 742, 950, 510, 915, 546, 0, 0, 0, 0, 0, 0, 0, 0, 490, 1225, 1037, 548, 259, 951, 553, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 672, 646, 672, 758, 488, 461, 940, 820, 448, 96, 0, 0, 0, 0, 0, 0, 772, 849, 548, 922, 847, 702, 926, 513, 0, 0, 0, 0, 0, 0, 0, 0, 742, 755, 563, 591, 477, 373, 911, 1248, 572, 0, 0, 0, 0, 0, 0, 0, 717, 692, 544, 485, 1420, 882, 707, 455, 124, 120, 0, 0, 0, 0, 0, 0, 794, 527, 786, 867, 1310, 1018, 314, 669, 0, 0, 0, 0, 0, 0, 0, 0, 750, 450, 1336, 918, 738, 446, 750, 464, 369, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6023, 6066, 6001, 6079, 6232, 6146, 6285, 6320] avg=6144.0000 max=6320 max_rank=7 min=6001 min_rank=2 imbalance_ratio=1.028646 max_load_reduction=42832 max_load_reduction_pct=0.871419 moved_to_echo=24983 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1152, 1076, 467, 783, 655, 617, 942, 614, 0, 0, 0, 0, 0, 0, 0, 0, 593, 1015, 1041, 649, 226, 1026, 575, 996, 0, 0, 0, 0, 0, 0, 0, 0, 666, 617, 772, 759, 572, 392, 1031, 930, 462, 103, 0, 0, 0, 0, 0, 0, 644, 838, 537, 1019, 882, 603, 978, 538, 0, 0, 0, 0, 0, 0, 0, 0, 622, 917, 658, 586, 539, 343, 837, 1149, 616, 0, 0, 0, 0, 0, 0, 0, 698, 686, 443, 661, 1395, 896, 570, 452, 129, 117, 0, 0, 0, 0, 0, 0, 704, 557, 949, 753, 1280, 828, 329, 609, 0, 0, 0, 0, 0, 0, 0, 0, 736, 559, 1295, 955, 603, 346, 747, 330, 386, 102, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6306, 6121, 6304, 6039, 6267, 6047, 6009, 6059] avg=6144.0000 max=6306 max_rank=0 min=6009 min_rank=6 imbalance_ratio=1.026367 max_load_reduction=42846 max_load_reduction_pct=0.871704 moved_to_echo=24382 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=2 rerouted_tokens_per_expert_current_ep_rank=[1111, 1011, 510, 748, 707, 617, 913, 549, 0, 0, 0, 0, 0, 0, 0, 0, 543, 1122, 956, 616, 234, 966, 711, 1065, 0, 0, 0, 0, 0, 0, 0, 0, 634, 668, 714, 732, 547, 398, 915, 857, 448, 97, 0, 0, 0, 0, 0, 0, 703, 1209, 525, 963, 1003, 688, 811, 534, 0, 0, 0, 0, 0, 0, 0, 0, 474, 741, 608, 628, 502, 426, 971, 1228, 581, 0, 0, 0, 0, 0, 0, 0, 792, 720, 513, 499, 1437, 837, 690, 403, 128, 83, 0, 0, 0, 0, 0, 0, 705, 538, 1037, 852, 914, 925, 344, 598, 0, 0, 0, 0, 0, 0, 0, 0, 687, 518, 1237, 991, 701, 369, 742, 426, 383, 99, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6166, 6213, 6010, 6436, 6159, 6102, 5913, 6153] avg=6144.0000 max=6436 max_rank=3 min=5913 min_rank=6 imbalance_ratio=1.047526 max_load_reduction=42716 max_load_reduction_pct=0.869059 moved_to_echo=24327 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 5760, 4992, 4864, 3968, 3200, 7808, 9728, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5760, 7424, 4224, 7936, 7168, 5632, 7168, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6144, 4352, 7168, 7040, 9984, 7168, 2816, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8832, 7168, 4224, 6272, 6528, 4736, 7296, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 5376, 5888, 6144, 4224, 3584, 7936, 6912, 3584, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[5760, 4352, 10240, 7552, 5376, 3200, 6272, 3328, 3072, 896, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 9472, 7808, 4736, 2048, 7808, 5120, 8448, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 5632, 3968, 4608, 11264, 7296, 5632, 3584, 1024, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=50048 slack_slots=256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=3 tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 330, 845, 1407, 575, 1301, 1140, 926, 1042, 731, 555, 759, 488, 900, 840, 1057, 506, 841, 913, 594, 593, 631, 724, 1526, 495, 1166, 537, 656, 338, 720, 524, 391, 1069, 1276, 421, 330, 820, 561, 867, 294, 425, 2259, 957, 879, 561, 571, 759, 748, 1082, 865, 519, 773, 598, 725, 873, 774, 608] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=3 tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 417, 850, 1357, 592, 1199, 924, 1030, 1008, 671, 716, 696, 631, 1088, 871, 1065, 699, 847, 818, 579, 464, 770, 621, 1312, 430, 1236, 377, 642, 500, 681, 384, 460, 1045, 1033, 389, 456, 697, 683, 880, 320, 347, 1988, 1344, 1001, 441, 572, 818, 619, 894, 948, 621, 770, 606, 806, 860, 931, 685] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=3 tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 466, 922, 1314, 532, 1008, 823, 1038, 1024, 699, 447, 750, 818, 1093, 874, 1205, 528, 835, 1008, 648, 483, 752, 664, 1403, 317, 1126, 526, 590, 468, 785, 419, 338, 972, 1077, 411, 382, 759, 474, 864, 244, 346, 2091, 1075, 984, 383, 605, 784, 661, 967, 882, 555, 867, 616, 883, 854, 928, 736] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=3 tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 436, 921, 1336, 533, 961, 830, 1091, 1147, 862, 427, 683, 553, 1150, 902, 942, 918, 805, 750, 610, 456, 746, 656, 1327, 369, 1143, 518, 735, 428, 607, 485, 517, 886, 1062, 509, 260, 658, 615, 939, 410, 306, 2189, 1342, 976, 344, 548, 870, 702, 913, 976, 602, 811, 728, 627, 932, 915, 714] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=3 tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 407, 939, 1397, 549, 1252, 885, 960, 1234, 703, 613, 652, 848, 1001, 826, 1098, 599, 862, 792, 659, 553, 892, 648, 1416, 393, 1058, 522, 632, 607, 780, 381, 382, 1047, 1285, 434, 519, 823, 592, 881, 287, 482, 2023, 1024, 888, 299, 486, 884, 637, 1066, 859, 486, 749, 713, 709, 739, 961, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=3 tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 375, 877, 1387, 634, 1079, 853, 1180, 1009, 751, 531, 731, 771, 995, 782, 1165, 639, 791, 680, 600, 548, 582, 667, 1306, 429, 1276, 461, 698, 496, 742, 480, 416, 997, 919, 432, 443, 788, 531, 915, 306, 324, 2016, 1176, 847, 359, 790, 759, 655, 1160, 1167, 556, 802, 602, 743, 924, 720, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=3 tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 359, 964, 1419, 523, 1182, 852, 950, 1099, 633, 574, 684, 788, 1050, 700, 1160, 745, 789, 744, 573, 532, 749, 718, 1486, 464, 1222, 435, 679, 475, 823, 453, 391, 997, 1335, 475, 439, 727, 615, 847, 294, 424, 1933, 1194, 830, 435, 536, 829, 653, 1007, 866, 503, 818, 699, 640, 801, 895, 547] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=3 tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 405, 941, 1409, 635, 1246, 856, 1000, 1041, 715, 511, 746, 617, 1059, 980, 1036, 639, 944, 811, 710, 495, 747, 647, 1404, 397, 1063, 482, 668, 436, 767, 504, 354, 926, 1197, 478, 496, 695, 512, 809, 343, 399, 2038, 1074, 930, 308, 638, 874, 603, 977, 924, 582, 733, 650, 628, 962, 1019, 758] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=3 rerouted_tokens_per_expert_current_ep_rank=[359, 597, 622, 761, 941, 650, 575, 969, 623, 0, 0, 0, 0, 0, 0, 0, 436, 921, 117, 533, 961, 830, 1091, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 862, 427, 683, 553, 1150, 902, 801, 918, 0, 0, 0, 0, 0, 0, 0, 0, 805, 750, 610, 456, 746, 656, 1327, 369, 78, 49, 0, 0, 0, 0, 0, 0, 1143, 518, 735, 428, 607, 485, 517, 886, 547, 226, 0, 0, 0, 0, 0, 0, 1062, 509, 260, 658, 615, 939, 410, 306, 1322, 0, 0, 0, 0, 0, 0, 0, 641, 1342, 976, 344, 548, 870, 702, 913, 0, 0, 0, 0, 0, 0, 0, 0, 976, 602, 811, 728, 627, 932, 915, 714, 63, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6097, 6036, 6296, 5846, 6092, 6081, 6336, 6368] avg=6144.0000 max=6368 max_rank=7 min=5846 min_rank=3 imbalance_ratio=1.036458 max_load_reduction=42784 max_load_reduction_pct=0.870443 moved_to_echo=24877 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=3 rerouted_tokens_per_expert_current_ep_rank=[506, 566, 687, 613, 927, 713, 649, 802, 637, 0, 0, 0, 0, 0, 0, 0, 417, 850, 105, 592, 1199, 924, 1030, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 671, 716, 696, 631, 1088, 871, 897, 699, 0, 0, 0, 0, 0, 0, 0, 0, 847, 818, 579, 464, 770, 621, 1312, 430, 93, 53, 0, 0, 0, 0, 0, 0, 1236, 377, 642, 500, 681, 384, 460, 1045, 562, 210, 0, 0, 0, 0, 0, 0, 1033, 389, 456, 697, 683, 880, 320, 347, 1205, 0, 0, 0, 0, 0, 0, 0, 573, 1344, 1001, 441, 572, 818, 619, 894, 0, 0, 0, 0, 0, 0, 0, 0, 948, 621, 770, 606, 806, 860, 931, 685, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6100, 6125, 6269, 5987, 6097, 6010, 6262, 6302] avg=6144.0000 max=6302 max_rank=7 min=5987 min_rank=3 imbalance_ratio=1.025716 max_load_reduction=42850 max_load_reduction_pct=0.871785 moved_to_echo=24671 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=3 rerouted_tokens_per_expert_current_ep_rank=[470, 523, 603, 587, 938, 629, 697, 741, 651, 0, 0, 0, 0, 0, 0, 0, 407, 939, 123, 549, 1252, 885, 960, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 703, 613, 652, 848, 1001, 826, 934, 599, 0, 0, 0, 0, 0, 0, 0, 0, 862, 792, 659, 553, 892, 648, 1416, 393, 91, 51, 0, 0, 0, 0, 0, 0, 1058, 522, 632, 607, 780, 381, 382, 1047, 572, 209, 0, 0, 0, 0, 0, 0, 1285, 434, 519, 823, 592, 881, 287, 482, 1222, 0, 0, 0, 0, 0, 0, 0, 592, 1024, 888, 299, 486, 884, 637, 1066, 0, 0, 0, 0, 0, 0, 0, 0, 859, 486, 749, 713, 709, 739, 961, 551, 73, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5839, 6349, 6176, 6357, 6190, 6525, 5876, 5840] avg=6144.0000 max=6525 max_rank=5 min=5839 min_rank=0 imbalance_ratio=1.062012 max_load_reduction=42627 max_load_reduction_pct=0.867249 moved_to_echo=24431 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=3 rerouted_tokens_per_expert_current_ep_rank=[462, 370, 844, 668, 817, 869, 742, 715, 656, 0, 0, 0, 0, 0, 0, 0, 330, 845, 124, 575, 1301, 1140, 926, 1042, 0, 0, 0, 0, 0, 0, 0, 0, 731, 555, 759, 488, 900, 840, 899, 506, 0, 0, 0, 0, 0, 0, 0, 0, 841, 913, 594, 593, 631, 724, 1526, 495, 88, 51, 0, 0, 0, 0, 0, 0, 1166, 537, 656, 338, 720, 524, 391, 1069, 576, 233, 0, 0, 0, 0, 0, 0, 1276, 421, 330, 820, 561, 867, 294, 425, 1364, 0, 0, 0, 0, 0, 0, 0, 662, 957, 879, 561, 571, 759, 748, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 865, 519, 773, 598, 725, 873, 774, 608, 70, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6283, 5678, 6456, 6210, 6358, 6219, 5805] avg=6144.0000 max=6456 max_rank=3 min=5678 min_rank=2 imbalance_ratio=1.050781 max_load_reduction=42696 max_load_reduction_pct=0.868652 moved_to_echo=24592 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=3 rerouted_tokens_per_expert_current_ep_rank=[523, 658, 723, 635, 808, 728, 741, 782, 661, 0, 0, 0, 0, 0, 0, 0, 359, 964, 125, 523, 1182, 852, 950, 1099, 0, 0, 0, 0, 0, 0, 0, 0, 633, 574, 684, 788, 1050, 700, 986, 745, 0, 0, 0, 0, 0, 0, 0, 0, 789, 744, 573, 532, 749, 718, 1486, 464, 97, 52, 0, 0, 0, 0, 0, 0, 1222, 435, 679, 475, 823, 453, 391, 997, 581, 200, 0, 0, 0, 0, 0, 0, 1335, 475, 439, 727, 615, 847, 294, 424, 1167, 0, 0, 0, 0, 0, 0, 0, 566, 1194, 830, 435, 536, 829, 653, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 866, 503, 818, 699, 640, 801, 895, 547, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6259, 6054, 6160, 6204, 6256, 6323, 6050, 5846] avg=6144.0000 max=6323 max_rank=5 min=5846 min_rank=7 imbalance_ratio=1.029134 max_load_reduction=42829 max_load_reduction_pct=0.871358 moved_to_echo=24475 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=3 rerouted_tokens_per_expert_current_ep_rank=[449, 563, 694, 949, 814, 753, 630, 764, 646, 0, 0, 0, 0, 0, 0, 0, 375, 877, 122, 634, 1079, 853, 1180, 1009, 0, 0, 0, 0, 0, 0, 0, 0, 751, 531, 731, 771, 995, 782, 991, 639, 0, 0, 0, 0, 0, 0, 0, 0, 791, 680, 600, 548, 582, 667, 1306, 429, 97, 51, 0, 0, 0, 0, 0, 0, 1276, 461, 698, 496, 742, 480, 416, 997, 568, 208, 0, 0, 0, 0, 0, 0, 919, 432, 443, 788, 531, 915, 306, 324, 1217, 0, 0, 0, 0, 0, 0, 0, 591, 1176, 847, 359, 790, 759, 655, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 556, 802, 602, 743, 924, 720, 674, 77, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 6129, 6191, 5751, 6342, 5875, 6337, 6265] avg=6144.0000 max=6342 max_rank=4 min=5751 min_rank=3 imbalance_ratio=1.032227 max_load_reduction=42810 max_load_reduction_pct=0.870972 moved_to_echo=24819 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=3 rerouted_tokens_per_expert_current_ep_rank=[596, 615, 717, 744, 1206, 610, 599, 762, 612, 0, 0, 0, 0, 0, 0, 0, 466, 922, 116, 532, 1008, 823, 1038, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 699, 447, 750, 818, 1093, 874, 1025, 528, 0, 0, 0, 0, 0, 0, 0, 0, 835, 1008, 648, 483, 752, 664, 1403, 317, 100, 48, 0, 0, 0, 0, 0, 0, 1126, 526, 590, 468, 785, 419, 338, 972, 538, 216, 0, 0, 0, 0, 0, 0, 1077, 411, 382, 759, 474, 864, 244, 346, 1263, 0, 0, 0, 0, 0, 0, 0, 612, 1075, 984, 383, 605, 784, 661, 967, 0, 0, 0, 0, 0, 0, 0, 0, 882, 555, 867, 616, 883, 854, 928, 736, 80, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6461, 5929, 6234, 6258, 5978, 5820, 6071, 6401] avg=6144.0000 max=6461 max_rank=0 min=5820 min_rank=5 imbalance_ratio=1.051595 max_load_reduction=42691 max_load_reduction_pct=0.868551 moved_to_echo=24270 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=3 rerouted_tokens_per_expert_current_ep_rank=[408, 488, 720, 671, 895, 708, 668, 776, 657, 0, 0, 0, 0, 0, 0, 0, 405, 941, 123, 635, 1246, 856, 1000, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 715, 511, 746, 617, 1059, 980, 881, 639, 0, 0, 0, 0, 0, 0, 0, 0, 944, 811, 710, 495, 747, 647, 1404, 397, 86, 52, 0, 0, 0, 0, 0, 0, 1063, 482, 668, 436, 767, 504, 354, 926, 577, 210, 0, 0, 0, 0, 0, 0, 1197, 478, 496, 695, 512, 809, 343, 399, 1231, 0, 0, 0, 0, 0, 0, 0, 597, 1074, 930, 308, 638, 874, 603, 977, 0, 0, 0, 0, 0, 0, 0, 0, 924, 582, 733, 650, 628, 962, 1019, 758, 69, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5991, 6247, 6148, 6293, 5987, 6160, 6001, 6325] avg=6144.0000 max=6325 max_rank=7 min=5987 min_rank=4 imbalance_ratio=1.029460 max_load_reduction=42827 max_load_reduction_pct=0.871318 moved_to_echo=24473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=3 +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:336: UserWarning: The use_host_meta is deprecated, it will be removed in the future. + warnings.warn("The use_host_meta is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 6528, 4992, 4224, 5888, 5376, 11264, 3328, 768, 512, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 9216, 7424, 3200, 4864, 6656, 5376, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4480, 5632, 5632, 7424, 5760, 5376, 6400, 5248, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 4480, 6400, 5248, 5888, 7040, 7168, 5376, 640, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[9344, 3968, 5376, 3840, 6016, 3712, 3328, 8064, 4608, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3200, 7296, 1024, 4608, 9344, 7168, 8192, 8704, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9216, 3584, 3328, 6016, 4608, 7040, 2560, 3072, 10112, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5888, 4480, 5760, 5632, 8448, 6784, 7424, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=50048 slack_slots=256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=4 tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 558, 973, 438, 1507, 1006, 353, 970, 734, 1115, 654, 635, 951, 954, 413, 1366, 809, 772, 752, 1019, 1152, 616, 271, 1100, 136, 1398, 714, 1087, 774, 1052, 897, 708, 777, 801, 480, 431, 1003, 828, 1098, 639, 566, 415, 1303, 1505, 648, 832, 719, 371, 591, 1293, 919, 244, 478, 425, 885, 780, 817] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=4 tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 580, 876, 296, 1504, 1210, 823, 973, 862, 993, 563, 952, 1248, 798, 500, 729, 951, 554, 604, 1168, 608, 671, 312, 753, 291, 1404, 788, 1176, 608, 1080, 665, 688, 719, 838, 475, 350, 1282, 912, 1272, 577, 700, 362, 979, 1558, 707, 818, 721, 332, 545, 1174, 918, 587, 485, 429, 902, 737, 777] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=4 tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 560, 1066, 278, 1567, 965, 556, 948, 637, 1334, 609, 978, 1121, 837, 443, 950, 843, 620, 642, 1367, 1015, 880, 246, 532, 289, 1392, 772, 1494, 430, 1178, 718, 652, 1024, 807, 446, 350, 1597, 746, 825, 743, 908, 363, 866, 1487, 802, 967, 548, 375, 536, 1025, 837, 383, 411, 565, 832, 727, 674] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=4 tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 454, 982, 340, 1476, 1021, 871, 1005, 679, 1213, 634, 874, 1114, 882, 490, 901, 808, 622, 597, 1568, 727, 788, 328, 750, 206, 1277, 822, 1183, 934, 1082, 770, 784, 768, 875, 529, 278, 1197, 1095, 1010, 706, 554, 420, 1105, 1486, 545, 809, 608, 301, 648, 1143, 842, 208, 535, 540, 967, 644, 688] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=4 tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 513, 1060, 298, 1484, 962, 1076, 1056, 680, 1195, 661, 977, 1033, 726, 354, 909, 774, 561, 668, 1227, 789, 732, 410, 661, 195, 1471, 797, 1339, 755, 1313, 878, 777, 726, 837, 515, 363, 1267, 1041, 1207, 480, 788, 383, 973, 1160, 710, 898, 623, 388, 604, 1029, 911, 314, 507, 504, 870, 750, 797] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=4 tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 478, 1210, 383, 1621, 980, 429, 1046, 718, 1255, 608, 998, 906, 748, 420, 1029, 810, 670, 760, 1237, 1094, 844, 279, 800, 132, 1354, 810, 1334, 495, 1328, 830, 499, 787, 896, 454, 309, 1248, 1069, 1237, 555, 537, 378, 1011, 1353, 584, 793, 678, 397, 657, 1314, 814, 247, 390, 473, 905, 735, 720] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=4 tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 476, 1039, 410, 1513, 1072, 923, 864, 712, 1049, 650, 1077, 866, 751, 588, 860, 926, 607, 775, 1185, 594, 752, 298, 643, 377, 1406, 1008, 1350, 776, 1227, 672, 691, 852, 706, 532, 354, 1213, 940, 848, 473, 954, 527, 919, 1525, 620, 927, 588, 438, 503, 899, 831, 348, 656, 547, 1008, 825, 701] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=4 tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 483, 947, 341, 1486, 1008, 1044, 982, 564, 1159, 565, 1074, 1130, 664, 345, 1073, 893, 543, 717, 1194, 781, 973, 255, 708, 310, 1399, 862, 1145, 860, 955, 819, 608, 948, 902, 461, 249, 1109, 939, 912, 495, 692, 468, 1107, 1555, 576, 904, 515, 351, 535, 847, 830, 370, 663, 483, 1006, 713, 681] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=4 rerouted_tokens_per_expert_current_ep_rank=[591, 991, 296, 428, 595, 433, 577, 478, 1392, 269, 0, 0, 0, 0, 0, 0, 560, 1066, 278, 823, 965, 556, 948, 637, 0, 0, 0, 0, 0, 0, 0, 0, 536, 609, 978, 1121, 837, 443, 950, 843, 0, 0, 0, 0, 0, 0, 0, 0, 620, 642, 1367, 1015, 880, 246, 532, 289, 475, 249, 0, 0, 0, 0, 0, 0, 0, 772, 1401, 430, 1178, 718, 652, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 807, 446, 350, 1541, 746, 825, 743, 908, 0, 0, 0, 0, 0, 0, 0, 0, 363, 866, 1487, 802, 967, 548, 375, 536, 93, 56, 35, 0, 0, 0, 0, 0, 1025, 837, 383, 411, 565, 832, 727, 674, 514, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6050, 5833, 6317, 6315, 6175, 6366, 6128, 5968] avg=6144.0000 max=6366 max_rank=5 min=5833 min_rank=1 imbalance_ratio=1.036133 max_load_reduction=42786 max_load_reduction_pct=0.870483 moved_to_echo=24637 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=4 rerouted_tokens_per_expert_current_ep_rank=[637, 849, 305, 566, 655, 316, 390, 702, 1398, 258, 0, 0, 0, 0, 0, 0, 558, 973, 438, 793, 1006, 353, 970, 734, 0, 0, 0, 0, 0, 0, 0, 0, 447, 654, 635, 951, 954, 413, 1366, 809, 0, 0, 0, 0, 0, 0, 0, 0, 772, 752, 1019, 1152, 616, 271, 1100, 136, 456, 208, 0, 0, 0, 0, 0, 0, 0, 714, 1019, 774, 1052, 897, 708, 777, 0, 0, 0, 0, 0, 0, 0, 0, 801, 480, 431, 968, 828, 1098, 639, 566, 0, 0, 0, 0, 0, 0, 0, 0, 415, 1303, 1505, 648, 832, 719, 371, 591, 68, 35, 30, 0, 0, 0, 0, 0, 1293, 919, 244, 478, 425, 885, 780, 817, 430, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6076, 5825, 6229, 6482, 5941, 5811, 6517, 6271] avg=6144.0000 max=6517 max_rank=6 min=5811 min_rank=5 imbalance_ratio=1.060710 max_load_reduction=42635 max_load_reduction_pct=0.867411 moved_to_echo=24540 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=4 rerouted_tokens_per_expert_current_ep_rank=[550, 958, 316, 467, 575, 360, 433, 517, 1471, 258, 0, 0, 0, 0, 0, 0, 513, 1060, 298, 772, 962, 1076, 1056, 680, 0, 0, 0, 0, 0, 0, 0, 0, 469, 661, 977, 1033, 726, 354, 909, 774, 0, 0, 0, 0, 0, 0, 0, 0, 561, 668, 1227, 789, 732, 410, 661, 195, 454, 226, 0, 0, 0, 0, 0, 0, 0, 797, 1252, 755, 1313, 878, 777, 726, 0, 0, 0, 0, 0, 0, 0, 0, 837, 515, 363, 1219, 1041, 1207, 480, 788, 0, 0, 0, 0, 0, 0, 0, 0, 383, 973, 1160, 710, 898, 623, 388, 604, 87, 48, 36, 0, 0, 0, 0, 0, 1029, 911, 314, 507, 504, 870, 750, 797, 464, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5905, 6417, 5903, 5923, 6498, 6450, 5910, 6146] avg=6144.0000 max=6498 max_rank=4 min=5903 min_rank=2 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=4 rerouted_tokens_per_expert_current_ep_rank=[694, 958, 444, 505, 606, 397, 472, 692, 1404, 258, 0, 0, 0, 0, 0, 0, 580, 876, 296, 791, 1210, 823, 973, 862, 0, 0, 0, 0, 0, 0, 0, 0, 399, 563, 952, 1248, 798, 500, 729, 951, 0, 0, 0, 0, 0, 0, 0, 0, 554, 604, 1168, 608, 671, 312, 753, 291, 455, 185, 0, 0, 0, 0, 0, 0, 0, 788, 1103, 608, 1080, 665, 688, 719, 0, 0, 0, 0, 0, 0, 0, 0, 838, 475, 350, 1238, 912, 1272, 577, 700, 0, 0, 0, 0, 0, 0, 0, 0, 362, 979, 1558, 707, 818, 721, 332, 545, 73, 44, 26, 0, 0, 0, 0, 0, 1174, 918, 587, 485, 429, 902, 737, 777, 383, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6430, 6411, 6140, 5601, 5651, 6362, 6165, 6392] avg=6144.0000 max=6430 max_rank=0 min=5601 min_rank=3 imbalance_ratio=1.046549 max_load_reduction=42722 max_load_reduction_pct=0.869181 moved_to_echo=24570 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=4 rerouted_tokens_per_expert_current_ep_rank=[654, 876, 367, 385, 453, 370, 534, 642, 1406, 259, 0, 0, 0, 0, 0, 0, 476, 1039, 410, 796, 1072, 923, 864, 712, 0, 0, 0, 0, 0, 0, 0, 0, 422, 650, 1077, 866, 751, 588, 860, 926, 0, 0, 0, 0, 0, 0, 0, 0, 607, 775, 1185, 594, 752, 298, 643, 377, 458, 195, 0, 0, 0, 0, 0, 0, 0, 1008, 1266, 776, 1227, 672, 691, 852, 0, 0, 0, 0, 0, 0, 0, 0, 706, 532, 354, 1171, 940, 848, 473, 954, 0, 0, 0, 0, 0, 0, 0, 0, 527, 919, 1525, 620, 927, 588, 438, 503, 84, 42, 28, 0, 0, 0, 0, 0, 899, 831, 348, 656, 547, 1008, 825, 701, 404, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5946, 6292, 6140, 5884, 6492, 5978, 6201, 6219] avg=6144.0000 max=6492 max_rank=4 min=5884 min_rank=3 imbalance_ratio=1.056641 max_load_reduction=42660 max_load_reduction_pct=0.867920 moved_to_echo=24890 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=4 rerouted_tokens_per_expert_current_ep_rank=[647, 855, 372, 525, 551, 573, 600, 831, 1399, 255, 0, 0, 0, 0, 0, 0, 483, 947, 341, 781, 1008, 1044, 982, 564, 0, 0, 0, 0, 0, 0, 0, 0, 465, 565, 1074, 1130, 664, 345, 1073, 893, 0, 0, 0, 0, 0, 0, 0, 0, 543, 717, 1194, 781, 973, 255, 708, 310, 450, 216, 0, 0, 0, 0, 0, 0, 0, 862, 1074, 860, 955, 819, 608, 948, 0, 0, 0, 0, 0, 0, 0, 0, 902, 461, 249, 1071, 939, 912, 495, 692, 0, 0, 0, 0, 0, 0, 0, 0, 468, 1107, 1555, 576, 904, 515, 351, 535, 71, 38, 31, 0, 0, 0, 0, 0, 847, 830, 370, 663, 483, 1006, 713, 681, 447, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6608, 6150, 6209, 6147, 6126, 5721, 6151, 6040] avg=6144.0000 max=6608 max_rank=0 min=5721 min_rank=5 imbalance_ratio=1.075521 max_load_reduction=42544 max_load_reduction_pct=0.865560 moved_to_echo=24038 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=4 rerouted_tokens_per_expert_current_ep_rank=[592, 883, 436, 547, 467, 501, 401, 679, 1354, 278, 0, 0, 0, 0, 0, 0, 478, 1210, 383, 852, 980, 429, 1046, 718, 0, 0, 0, 0, 0, 0, 0, 0, 504, 608, 998, 906, 748, 420, 1029, 810, 0, 0, 0, 0, 0, 0, 0, 0, 670, 760, 1237, 1094, 844, 279, 800, 132, 491, 234, 0, 0, 0, 0, 0, 0, 0, 810, 1251, 495, 1328, 830, 499, 787, 0, 0, 0, 0, 0, 0, 0, 0, 896, 454, 309, 1205, 1069, 1237, 555, 537, 0, 0, 0, 0, 0, 0, 0, 0, 378, 1011, 1353, 584, 793, 678, 397, 657, 83, 43, 33, 0, 0, 0, 0, 0, 1314, 814, 247, 390, 473, 905, 735, 720, 484, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6138, 6096, 6023, 6541, 6000, 6262, 6010, 6082] avg=6144.0000 max=6541 max_rank=3 min=6000 min_rank=4 imbalance_ratio=1.064616 max_load_reduction=42611 max_load_reduction_pct=0.866923 moved_to_echo=24354 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=4 rerouted_tokens_per_expert_current_ep_rank=[535, 856, 467, 449, 537, 457, 353, 815, 1277, 253, 0, 0, 0, 0, 0, 0, 454, 982, 340, 776, 1021, 871, 1005, 679, 0, 0, 0, 0, 0, 0, 0, 0, 488, 634, 874, 1114, 882, 490, 901, 808, 0, 0, 0, 0, 0, 0, 0, 0, 622, 597, 1568, 727, 788, 328, 750, 206, 447, 226, 0, 0, 0, 0, 0, 0, 0, 822, 1109, 934, 1082, 770, 784, 768, 0, 0, 0, 0, 0, 0, 0, 0, 875, 529, 278, 1155, 1095, 1010, 706, 554, 0, 0, 0, 0, 0, 0, 0, 0, 420, 1105, 1486, 545, 809, 608, 301, 648, 74, 42, 32, 0, 0, 0, 0, 0, 1143, 842, 208, 535, 540, 967, 644, 688, 467, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5999, 6128, 6191, 6259, 6269, 6202, 6070, 6034] avg=6144.0000 max=6269 max_rank=4 min=5999 min_rank=0 imbalance_ratio=1.020345 max_load_reduction=42883 max_load_reduction_pct=0.872457 moved_to_echo=24575 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[8832, 6912, 2816, 4224, 3968, 7424, 6016, 5888, 3712, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 7296, 3072, 3968, 4480, 3456, 3840, 5376, 11136, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3840, 4992, 7680, 8448, 6400, 3584, 7936, 6912, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 8192, 2816, 6400, 8320, 6144, 7936, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 3968, 2688, 9600, 7680, 8448, 4736, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 8320, 11648, 5248, 7040, 5120, 3072, 4736, 640, 384, 256, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4992, 5632, 9984, 6784, 6272, 2432, 6016, 2048, 3712, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[0, 6656, 9600, 5632, 9216, 6272, 5504, 6656, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=5 tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 1366, 925, 376, 285, 321, 909, 841, 460, 573, 982, 1001, 651, 914, 707, 919, 764, 1160, 675, 469, 968, 1042, 669, 1350, 809, 360, 364, 974, 504, 1212, 371, 500, 419, 386, 437, 964, 1160, 570, 962, 672, 767, 638, 879, 1005, 2291, 478, 214, 621, 646, 620, 1053, 387, 594, 390, 871, 1860, 813, 533] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=5 tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 1310, 507, 429, 316, 297, 929, 848, 987, 488, 1039, 873, 1141, 617, 674, 918, 623, 382, 796, 452, 1102, 1004, 559, 1524, 793, 852, 515, 845, 487, 955, 735, 491, 496, 354, 507, 883, 777, 639, 940, 728, 925, 771, 1220, 851, 1963, 713, 245, 473, 638, 800, 973, 437, 546, 183, 1071, 1536, 1043, 472] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=5 tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 1329, 791, 402, 368, 265, 876, 769, 637, 485, 1262, 1096, 821, 941, 586, 932, 655, 829, 559, 510, 1130, 990, 679, 1478, 749, 427, 320, 659, 477, 1070, 645, 629, 427, 375, 590, 738, 970, 701, 840, 532, 777, 910, 1420, 850, 1614, 605, 253, 440, 771, 822, 1170, 404, 510, 332, 1079, 1588, 744, 522] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=5 tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 1580, 783, 383, 428, 317, 856, 914, 753, 639, 1466, 827, 699, 669, 523, 929, 431, 702, 524, 453, 1120, 1052, 722, 1316, 877, 579, 305, 932, 435, 1533, 485, 401, 519, 539, 444, 805, 586, 818, 777, 771, 808, 659, 897, 1106, 2598, 615, 257, 652, 750, 671, 1015, 272, 508, 317, 790, 1768, 870, 570] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=5 tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 1235, 901, 297, 257, 364, 790, 813, 650, 739, 1186, 1316, 720, 838, 556, 1085, 587, 759, 818, 288, 839, 1195, 545, 1407, 955, 378, 361, 828, 502, 1238, 771, 479, 511, 361, 402, 722, 902, 517, 717, 669, 884, 692, 1081, 960, 2456, 720, 116, 552, 750, 597, 1346, 413, 438, 299, 736, 1502, 845, 437] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=5 tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 1484, 896, 530, 339, 220, 916, 862, 514, 498, 954, 1138, 779, 618, 591, 819, 526, 549, 523, 395, 1081, 1225, 1018, 1327, 810, 621, 576, 843, 492, 1278, 509, 401, 470, 402, 428, 1150, 770, 766, 870, 783, 795, 816, 663, 1058, 2230, 439, 207, 727, 1061, 801, 1108, 327, 670, 439, 535, 1787, 999, 632] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=5 tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 1673, 652, 395, 295, 209, 954, 762, 619, 537, 1154, 1099, 781, 689, 811, 707, 635, 1008, 475, 461, 1244, 1125, 958, 1439, 878, 452, 287, 794, 427, 1385, 654, 705, 612, 287, 359, 708, 909, 665, 842, 586, 827, 715, 981, 951, 2133, 448, 211, 663, 807, 893, 1057, 274, 592, 271, 727, 1575, 531, 516] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=5 tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 1422, 819, 495, 326, 643, 918, 738, 492, 583, 1009, 1008, 926, 816, 551, 1004, 708, 1039, 609, 297, 1155, 1085, 605, 1639, 854, 761, 633, 832, 504, 1228, 430, 458, 397, 422, 473, 1061, 860, 490, 712, 739, 904, 654, 935, 909, 1473, 592, 219, 704, 741, 557, 1150, 395, 466, 348, 937, 1442, 890, 434] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=5 rerouted_tokens_per_expert_current_ep_rank=[684, 653, 734, 522, 1447, 319, 1121, 588, 0, 0, 0, 0, 0, 0, 0, 0, 507, 429, 316, 297, 929, 848, 987, 488, 756, 518, 0, 0, 0, 0, 0, 0, 518, 873, 1141, 617, 674, 918, 623, 382, 0, 0, 0, 0, 0, 0, 0, 0, 796, 452, 1102, 1004, 559, 874, 793, 852, 0, 0, 0, 0, 0, 0, 0, 0, 515, 845, 487, 955, 735, 491, 496, 354, 650, 348, 204, 0, 0, 0, 0, 0, 507, 883, 777, 639, 940, 728, 925, 771, 173, 21, 0, 0, 0, 0, 0, 0, 1220, 851, 1207, 713, 245, 473, 638, 800, 0, 0, 0, 0, 0, 0, 0, 0, 973, 437, 546, 183, 1071, 1515, 1043, 472, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6068, 6075, 5746, 6432, 6080, 6364, 6147, 6240] avg=6144.0000 max=6432 max_rank=3 min=5746 min_rank=2 imbalance_ratio=1.046875 max_load_reduction=42720 max_load_reduction_pct=0.869141 moved_to_echo=24831 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=5 rerouted_tokens_per_expert_current_ep_rank=[669, 592, 787, 461, 967, 530, 1131, 708, 0, 0, 0, 0, 0, 0, 0, 0, 783, 383, 428, 317, 856, 914, 753, 639, 1001, 625, 0, 0, 0, 0, 0, 0, 731, 827, 699, 669, 523, 929, 431, 702, 0, 0, 0, 0, 0, 0, 0, 0, 524, 453, 1120, 1052, 722, 755, 877, 579, 0, 0, 0, 0, 0, 0, 0, 0, 305, 932, 435, 1533, 485, 401, 519, 539, 561, 491, 247, 0, 0, 0, 0, 0, 444, 805, 586, 818, 777, 771, 808, 659, 244, 25, 0, 0, 0, 0, 0, 0, 897, 1106, 1597, 615, 257, 652, 750, 671, 0, 0, 0, 0, 0, 0, 0, 0, 1015, 272, 508, 317, 790, 1743, 870, 570, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5845, 6699, 5511, 6082, 6448, 5937, 6545, 6085] avg=6144.0000 max=6699 max_rank=1 min=5511 min_rank=2 imbalance_ratio=1.090332 max_load_reduction=42453 max_load_reduction_pct=0.863708 moved_to_echo=25015 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=5 rerouted_tokens_per_expert_current_ep_rank=[730, 705, 911, 453, 1050, 537, 1115, 613, 0, 0, 0, 0, 0, 0, 0, 0, 925, 376, 285, 321, 909, 841, 460, 573, 883, 540, 0, 0, 0, 0, 0, 0, 489, 1001, 651, 914, 707, 919, 764, 1160, 0, 0, 0, 0, 0, 0, 0, 0, 675, 469, 968, 1042, 669, 774, 809, 360, 0, 0, 0, 0, 0, 0, 0, 0, 364, 974, 504, 1212, 371, 500, 419, 386, 576, 329, 213, 0, 0, 0, 0, 0, 437, 964, 1160, 570, 962, 672, 767, 638, 164, 26, 0, 0, 0, 0, 0, 0, 879, 1005, 1408, 478, 214, 621, 646, 620, 0, 0, 0, 0, 0, 0, 0, 0, 1053, 387, 594, 390, 871, 1834, 813, 533, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6114, 6113, 6605, 5766, 5848, 6360, 5871, 6475] avg=6144.0000 max=6605 max_rank=2 min=5766 min_rank=3 imbalance_ratio=1.075033 max_load_reduction=42547 max_load_reduction_pct=0.865621 moved_to_echo=24554 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=5 rerouted_tokens_per_expert_current_ep_rank=[582, 607, 748, 312, 933, 637, 1068, 666, 0, 0, 0, 0, 0, 0, 0, 0, 896, 530, 339, 220, 916, 862, 514, 498, 859, 587, 0, 0, 0, 0, 0, 0, 476, 1138, 779, 618, 591, 819, 526, 549, 0, 0, 0, 0, 0, 0, 0, 0, 523, 395, 1081, 1225, 1018, 761, 810, 621, 0, 0, 0, 0, 0, 0, 0, 0, 576, 843, 492, 1278, 509, 401, 470, 402, 566, 319, 231, 0, 0, 0, 0, 0, 428, 1150, 770, 766, 870, 783, 795, 816, 159, 25, 0, 0, 0, 0, 0, 0, 663, 1058, 1371, 439, 207, 727, 1061, 801, 0, 0, 0, 0, 0, 0, 0, 0, 1108, 327, 670, 439, 535, 1762, 999, 632, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5553, 6221, 5496, 6434, 6087, 6562, 6327, 6472] avg=6144.0000 max=6562 max_rank=5 min=5496 min_rank=2 imbalance_ratio=1.068034 max_load_reduction=42590 max_load_reduction_pct=0.866496 moved_to_echo=25448 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=5 rerouted_tokens_per_expert_current_ep_rank=[690, 712, 1004, 447, 1242, 676, 1031, 597, 0, 0, 0, 0, 0, 0, 0, 0, 791, 402, 368, 265, 876, 769, 637, 485, 622, 525, 0, 0, 0, 0, 0, 0, 629, 1096, 821, 941, 586, 932, 655, 829, 0, 0, 0, 0, 0, 0, 0, 0, 559, 510, 1130, 990, 679, 848, 749, 427, 0, 0, 0, 0, 0, 0, 0, 0, 320, 659, 477, 1070, 645, 629, 427, 375, 630, 423, 207, 0, 0, 0, 0, 0, 590, 738, 970, 701, 840, 532, 777, 910, 210, 22, 0, 0, 0, 0, 0, 0, 1420, 850, 992, 605, 253, 440, 771, 822, 0, 0, 0, 0, 0, 0, 0, 0, 1170, 404, 510, 332, 1079, 1566, 744, 522, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6399, 5740, 6489, 5892, 5862, 6290, 6153, 6327] avg=6144.0000 max=6489 max_rank=2 min=5740 min_rank=1 imbalance_ratio=1.056152 max_load_reduction=42663 max_load_reduction_pct=0.867981 moved_to_echo=24632 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=5 rerouted_tokens_per_expert_current_ep_rank=[777, 597, 1189, 426, 1118, 479, 1162, 750, 0, 0, 0, 0, 0, 0, 0, 0, 652, 395, 295, 209, 954, 762, 619, 537, 822, 662, 0, 0, 0, 0, 0, 0, 576, 1099, 781, 689, 811, 707, 635, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 475, 461, 1244, 1125, 958, 825, 878, 452, 0, 0, 0, 0, 0, 0, 0, 0, 287, 794, 427, 1385, 654, 705, 612, 287, 614, 386, 261, 0, 0, 0, 0, 0, 359, 708, 909, 665, 842, 586, 827, 715, 192, 22, 0, 0, 0, 0, 0, 0, 981, 951, 1311, 448, 211, 663, 807, 893, 0, 0, 0, 0, 0, 0, 0, 0, 1057, 274, 592, 271, 727, 1553, 531, 516, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6498, 5907, 6306, 6418, 6412, 5825, 6265, 5521] avg=6144.0000 max=6498 max_rank=0 min=5521 min_rank=7 imbalance_ratio=1.057617 max_load_reduction=42654 max_load_reduction_pct=0.867798 moved_to_echo=24023 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=5 rerouted_tokens_per_expert_current_ep_rank=[779, 511, 886, 543, 1176, 639, 1127, 630, 0, 0, 0, 0, 0, 0, 0, 0, 819, 495, 326, 643, 918, 738, 492, 583, 572, 566, 0, 0, 0, 0, 0, 0, 495, 1008, 926, 816, 551, 1004, 708, 1039, 0, 0, 0, 0, 0, 0, 0, 0, 609, 297, 1155, 1085, 605, 937, 854, 761, 0, 0, 0, 0, 0, 0, 0, 0, 633, 832, 504, 1228, 430, 458, 397, 422, 702, 342, 226, 0, 0, 0, 0, 0, 473, 1061, 860, 490, 712, 739, 904, 654, 172, 25, 0, 0, 0, 0, 0, 0, 935, 909, 901, 592, 219, 704, 741, 557, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 395, 466, 348, 937, 1417, 890, 434, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6291, 6152, 6547, 6303, 6174, 6090, 5558, 6037] avg=6144.0000 max=6547 max_rank=2 min=5558 min_rank=6 imbalance_ratio=1.065592 max_load_reduction=42605 max_load_reduction_pct=0.866801 moved_to_echo=23859 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=5 rerouted_tokens_per_expert_current_ep_rank=[735, 687, 587, 548, 1425, 690, 1158, 554, 0, 0, 0, 0, 0, 0, 0, 0, 901, 297, 257, 364, 790, 813, 650, 739, 946, 488, 0, 0, 0, 0, 0, 0, 591, 1316, 720, 838, 556, 1085, 587, 759, 0, 0, 0, 0, 0, 0, 0, 0, 818, 288, 839, 1195, 545, 807, 955, 378, 0, 0, 0, 0, 0, 0, 0, 0, 361, 828, 502, 1238, 771, 479, 511, 361, 600, 397, 193, 0, 0, 0, 0, 0, 402, 722, 902, 517, 717, 669, 884, 692, 198, 21, 0, 0, 0, 0, 0, 0, 1081, 960, 1510, 720, 116, 552, 750, 597, 0, 0, 0, 0, 0, 0, 0, 0, 1346, 413, 438, 299, 736, 1481, 845, 437, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6384, 6245, 6452, 5825, 6241, 5724, 6286, 5995] avg=6144.0000 max=6452 max_rank=2 min=5724 min_rank=5 imbalance_ratio=1.050130 max_load_reduction=42700 max_load_reduction_pct=0.868734 moved_to_echo=24246 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 6784, 3840, 9984, 4608, 4096, 3968, 3200, 4992, 3072, 1792, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 3328, 2688, 2688, 7168, 6656, 5120, 4608, 6528, 4608, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8192, 7808, 10368, 4736, 1792, 4864, 6272, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 7040, 7040, 5248, 6784, 5504, 6784, 5888, 1536, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 5120, 6912, 3712, 9472, 4608, 8960, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 8448, 6528, 6144, 5120, 7424, 4992, 6528, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4992, 3328, 8704, 8832, 5760, 6656, 6784, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8960, 2944, 4352, 2688, 6784, 12928, 6784, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=6 tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 1632, 873, 615, 690, 685, 566, 695, 535, 494, 335, 558, 445, 1179, 171, 1829, 669, 382, 945, 725, 505, 484, 418, 1051, 754, 1964, 492, 626, 547, 264, 420, 1800, 1169, 324, 425, 981, 754, 1050, 146, 509, 524, 635, 1071, 480, 366, 1441, 1677, 344, 442, 1426, 1394, 700, 348, 1117, 476, 1037, 531, 1242, 682, 496] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=6 tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 1690, 978, 552, 740, 626, 829, 622, 647, 411, 472, 336, 394, 1358, 304, 1755, 1030, 629, 568, 731, 673, 377, 964, 977, 591, 1916, 699, 826, 339, 519, 430, 1135, 712, 491, 356, 575, 745, 713, 168, 531, 602, 463, 802, 697, 311, 1560, 1199, 316, 658, 1341, 1581, 709, 409, 1602, 511, 995, 658, 581, 768, 309] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=6 tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 2001, 1106, 848, 739, 660, 662, 634, 502, 683, 607, 468, 525, 1264, 373, 1593, 1069, 621, 790, 506, 503, 405, 1030, 1056, 470, 2214, 576, 692, 635, 300, 539, 1098, 944, 504, 555, 693, 541, 795, 224, 669, 597, 665, 719, 307, 324, 1249, 1326, 352, 480, 1198, 1074, 724, 467, 1131, 512, 1573, 533, 672, 642, 623] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=6 tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 1952, 1096, 890, 811, 637, 713, 502, 612, 588, 488, 371, 406, 1108, 346, 1358, 680, 801, 524, 1129, 680, 258, 654, 1165, 562, 1723, 808, 731, 605, 325, 987, 733, 1011, 337, 455, 795, 783, 631, 373, 1025, 1070, 476, 376, 536, 317, 1565, 1507, 326, 574, 1230, 1183, 682, 456, 1706, 458, 984, 489, 717, 785, 336] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=6 tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 1779, 868, 661, 624, 693, 599, 679, 600, 639, 493, 642, 577, 1229, 303, 1607, 832, 703, 860, 864, 659, 375, 975, 1072, 653, 2073, 452, 724, 327, 428, 694, 1321, 941, 325, 504, 783, 640, 564, 162, 661, 503, 473, 783, 424, 382, 1616, 1556, 354, 496, 1685, 1119, 707, 240, 1072, 520, 1277, 472, 961, 455, 308] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=6 tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 1662, 937, 655, 615, 652, 525, 984, 691, 579, 481, 315, 466, 1405, 275, 1981, 887, 907, 895, 849, 578, 380, 1171, 1263, 331, 1903, 484, 525, 272, 377, 500, 1602, 837, 296, 366, 527, 574, 535, 141, 593, 456, 509, 969, 250, 412, 1130, 1635, 263, 893, 1421, 1125, 720, 405, 1430, 411, 1524, 509, 482, 665, 418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=6 tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 1968, 988, 597, 672, 568, 1045, 694, 669, 531, 378, 414, 363, 1204, 332, 1669, 528, 601, 673, 717, 507, 368, 672, 1450, 715, 2000, 423, 796, 459, 467, 392, 1609, 707, 444, 420, 928, 654, 877, 356, 736, 672, 533, 936, 537, 377, 1100, 1389, 261, 742, 929, 1679, 713, 331, 1074, 452, 915, 769, 507, 586, 425] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=6 tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 1746, 1134, 465, 391, 912, 1130, 638, 619, 532, 302, 611, 401, 1161, 302, 1829, 607, 634, 701, 639, 500, 298, 1007, 1690, 420, 1582, 441, 780, 367, 520, 432, 1375, 577, 454, 566, 643, 466, 657, 308, 654, 690, 556, 1020, 313, 426, 1291, 1352, 331, 745, 1317, 1087, 766, 444, 1465, 495, 1461, 609, 658, 557, 715] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1213, 993, 810, 383, 491, 266, 1106, 848, 0, 0, 0, 0, 0, 0, 0, 0, 739, 660, 662, 634, 502, 683, 607, 468, 1033, 172, 0, 0, 0, 0, 0, 0, 525, 1264, 373, 1119, 1069, 621, 790, 506, 0, 0, 0, 0, 0, 0, 0, 0, 503, 405, 1030, 1056, 470, 1500, 576, 692, 0, 0, 0, 0, 0, 0, 0, 0, 635, 300, 539, 1098, 944, 504, 555, 693, 714, 155, 159, 0, 0, 0, 0, 0, 541, 795, 224, 669, 597, 665, 719, 307, 1563, 0, 0, 0, 0, 0, 0, 0, 324, 1249, 138, 352, 480, 1198, 1074, 724, 0, 0, 0, 0, 0, 0, 0, 0, 467, 1131, 512, 1573, 533, 672, 642, 623, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6110, 6160, 6267, 6232, 6296, 6080, 5539, 6468] avg=6144.0000 max=6468 max_rank=7 min=5539 min_rank=6 imbalance_ratio=1.052734 max_load_reduction=42684 max_load_reduction_pct=0.868408 moved_to_echo=24383 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1301, 1146, 924, 503, 797, 225, 978, 552, 0, 0, 0, 0, 0, 0, 0, 0, 740, 626, 829, 622, 647, 411, 472, 336, 934, 145, 0, 0, 0, 0, 0, 0, 394, 1358, 304, 1233, 1030, 629, 568, 731, 0, 0, 0, 0, 0, 0, 0, 0, 673, 377, 964, 977, 591, 1299, 699, 826, 0, 0, 0, 0, 0, 0, 0, 0, 339, 519, 430, 1135, 712, 491, 356, 575, 617, 140, 175, 0, 0, 0, 0, 0, 745, 713, 168, 531, 602, 463, 802, 697, 1320, 0, 0, 0, 0, 0, 0, 0, 311, 1560, 125, 316, 658, 1341, 1581, 709, 0, 0, 0, 0, 0, 0, 0, 0, 409, 1602, 511, 995, 658, 581, 768, 309, 347, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6426, 5762, 6247, 6406, 5489, 6041, 6601, 6180] avg=6144.0000 max=6601 max_rank=6 min=5489 min_rank=4 imbalance_ratio=1.074382 max_load_reduction=42551 max_load_reduction_pct=0.865702 moved_to_echo=24311 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=6 rerouted_tokens_per_expert_current_ep_rank=[926, 1081, 786, 301, 632, 260, 1096, 890, 0, 0, 0, 0, 0, 0, 0, 0, 811, 637, 713, 502, 612, 588, 488, 371, 1174, 168, 0, 0, 0, 0, 0, 0, 406, 1108, 346, 955, 680, 801, 524, 1129, 0, 0, 0, 0, 0, 0, 0, 0, 680, 258, 654, 1165, 562, 1168, 808, 731, 0, 0, 0, 0, 0, 0, 0, 0, 605, 325, 987, 733, 1011, 337, 455, 795, 555, 177, 135, 0, 0, 0, 0, 0, 783, 631, 373, 1025, 1070, 476, 376, 536, 1524, 0, 0, 0, 0, 0, 0, 0, 317, 1565, 156, 326, 574, 1230, 1183, 682, 0, 0, 0, 0, 0, 0, 0, 0, 456, 1706, 458, 984, 489, 717, 785, 336, 268, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5972, 6064, 5949, 6026, 6115, 6794, 6033, 6199] avg=6144.0000 max=6794 max_rank=5 min=5949 min_rank=2 imbalance_ratio=1.105794 max_load_reduction=42358 max_load_reduction_pct=0.861776 moved_to_echo=25141 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1005, 972, 772, 473, 795, 218, 873, 615, 0, 0, 0, 0, 0, 0, 0, 0, 690, 685, 566, 695, 535, 494, 335, 558, 1307, 140, 0, 0, 0, 0, 0, 0, 445, 1179, 171, 1286, 669, 382, 945, 725, 0, 0, 0, 0, 0, 0, 0, 0, 505, 484, 418, 1051, 754, 1331, 492, 626, 0, 0, 0, 0, 0, 0, 0, 0, 547, 264, 420, 1800, 1169, 324, 425, 981, 633, 197, 182, 0, 0, 0, 0, 0, 754, 1050, 146, 509, 524, 635, 1071, 480, 1274, 0, 0, 0, 0, 0, 0, 0, 366, 1441, 173, 344, 442, 1426, 1394, 700, 0, 0, 0, 0, 0, 0, 0, 0, 348, 1117, 476, 1037, 531, 1242, 682, 496, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5723, 6005, 5802, 5661, 6942, 6443, 6286, 6290] avg=6144.0000 max=6942 max_rank=4 min=5661 min_rank=3 imbalance_ratio=1.129883 max_load_reduction=42210 max_load_reduction_pct=0.858765 moved_to_echo=25961 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1212, 1284, 1010, 436, 692, 256, 988, 597, 0, 0, 0, 0, 0, 0, 0, 0, 672, 568, 1045, 694, 669, 531, 378, 414, 1087, 171, 0, 0, 0, 0, 0, 0, 363, 1204, 332, 1164, 528, 601, 673, 717, 0, 0, 0, 0, 0, 0, 0, 0, 507, 368, 672, 1450, 715, 1351, 423, 796, 0, 0, 0, 0, 0, 0, 0, 0, 459, 467, 392, 1609, 707, 444, 420, 928, 649, 167, 171, 0, 0, 0, 0, 0, 654, 877, 356, 736, 672, 533, 936, 537, 1541, 0, 0, 0, 0, 0, 0, 0, 377, 1100, 135, 261, 742, 929, 1679, 713, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1074, 452, 915, 769, 507, 586, 425, 334, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6475, 6229, 5582, 6282, 6413, 6842, 5936, 5393] avg=6144.0000 max=6842 max_rank=5 min=5393 min_rank=7 imbalance_ratio=1.113607 max_load_reduction=42310 max_load_reduction_pct=0.860799 moved_to_echo=24584 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1182, 1219, 759, 462, 542, 237, 868, 661, 0, 0, 0, 0, 0, 0, 0, 0, 624, 693, 599, 679, 600, 639, 493, 642, 1213, 153, 0, 0, 0, 0, 0, 0, 577, 1229, 303, 1130, 832, 703, 860, 864, 0, 0, 0, 0, 0, 0, 0, 0, 659, 375, 975, 1072, 653, 1405, 452, 724, 0, 0, 0, 0, 0, 0, 0, 0, 327, 428, 694, 1321, 941, 325, 504, 783, 668, 182, 160, 0, 0, 0, 0, 0, 640, 564, 162, 661, 503, 473, 783, 424, 1389, 0, 0, 0, 0, 0, 0, 0, 382, 1616, 161, 354, 496, 1685, 1119, 707, 0, 0, 0, 0, 0, 0, 0, 0, 240, 1072, 520, 1277, 472, 961, 455, 308, 317, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5930, 6335, 6498, 6315, 6333, 5599, 6520, 5622] avg=6144.0000 max=6520 max_rank=6 min=5599 min_rank=5 imbalance_ratio=1.061198 max_load_reduction=42632 max_load_reduction_pct=0.867350 moved_to_echo=24074 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1263, 1117, 1148, 541, 440, 221, 937, 655, 0, 0, 0, 0, 0, 0, 0, 0, 615, 652, 525, 984, 691, 579, 481, 315, 1274, 143, 0, 0, 0, 0, 0, 0, 466, 1405, 275, 1392, 887, 907, 895, 849, 0, 0, 0, 0, 0, 0, 0, 0, 578, 380, 1171, 1263, 331, 1290, 484, 525, 0, 0, 0, 0, 0, 0, 0, 0, 272, 377, 500, 1602, 837, 296, 366, 527, 613, 192, 198, 0, 0, 0, 0, 0, 574, 535, 141, 593, 456, 509, 969, 250, 1298, 0, 0, 0, 0, 0, 0, 0, 412, 1130, 169, 263, 893, 1421, 1125, 720, 0, 0, 0, 0, 0, 0, 0, 0, 405, 1430, 411, 1524, 509, 482, 665, 418, 391, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6322, 6259, 7076, 6022, 5780, 5325, 6133, 6235] avg=6144.0000 max=7076 max_rank=2 min=5325 min_rank=5 imbalance_ratio=1.151693 max_load_reduction=42076 max_load_reduction_pct=0.856038 moved_to_echo=23473 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=6 rerouted_tokens_per_expert_current_ep_rank=[1314, 1238, 905, 345, 561, 232, 1134, 465, 0, 0, 0, 0, 0, 0, 0, 0, 391, 912, 1130, 638, 619, 532, 302, 611, 1053, 150, 0, 0, 0, 0, 0, 0, 401, 1161, 302, 1286, 607, 634, 701, 639, 0, 0, 0, 0, 0, 0, 0, 0, 500, 298, 1007, 1690, 420, 1072, 441, 780, 0, 0, 0, 0, 0, 0, 0, 0, 367, 520, 432, 1375, 577, 454, 566, 643, 510, 158, 182, 0, 0, 0, 0, 0, 466, 657, 308, 654, 690, 556, 1020, 313, 1364, 0, 0, 0, 0, 0, 0, 0, 426, 1291, 141, 331, 745, 1317, 1087, 766, 0, 0, 0, 0, 0, 0, 0, 0, 444, 1465, 495, 1461, 609, 658, 557, 715, 361, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6194, 6338, 5731, 6208, 5784, 6028, 6104, 6765] avg=6144.0000 max=6765 max_rank=7 min=5731 min_rank=2 imbalance_ratio=1.101074 max_load_reduction=42387 max_load_reduction_pct=0.862366 moved_to_echo=24681 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 11008, 1280, 2560, 5120, 10624, 10368, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5376, 5504, 6144, 5504, 4992, 4480, 3584, 3840, 9088, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4608, 3072, 6912, 9728, 4608, 10496, 4480, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9472, 9088, 7168, 3456, 4992, 1920, 8064, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 10624, 3840, 9856, 4608, 5888, 5248, 3712, 2816, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 9984, 2432, 9600, 6400, 5376, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 5888, 1920, 5504, 5120, 4352, 6784, 3584, 11392, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 3200, 4480, 10752, 6912, 3200, 3712, 6016, 4992, 1408, 1408, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=7 tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 439, 380, 783, 409, 362, 609, 930, 291, 1816, 1124, 954, 532, 682, 2021, 672, 345, 1103, 946, 863, 512, 651, 803, 465, 281, 463, 180, 1162, 623, 746, 1214, 1523, 1064, 970, 769, 1103, 1115, 356, 504, 2246, 597, 761, 698, 1115, 644, 917, 490, 169, 496, 972, 1516, 616, 440, 915, 278, 677, 347] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=7 tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 1051, 376, 592, 314, 225, 458, 801, 242, 2106, 938, 1035, 790, 643, 2398, 754, 429, 728, 671, 843, 407, 667, 602, 348, 347, 663, 373, 934, 571, 660, 1287, 1223, 1024, 1013, 678, 913, 1168, 437, 562, 1585, 830, 613, 1397, 888, 848, 479, 433, 136, 625, 1114, 1428, 1102, 633, 637, 224, 783, 341] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=7 tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 819, 270, 704, 338, 210, 330, 911, 179, 2160, 1205, 1014, 598, 1151, 2046, 799, 961, 1097, 1194, 675, 377, 620, 803, 403, 538, 422, 371, 655, 531, 662, 1113, 1105, 996, 726, 320, 751, 1064, 452, 577, 1799, 1119, 625, 869, 1149, 691, 625, 580, 143, 472, 769, 1066, 713, 562, 804, 247, 906, 586] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=7 tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 789, 510, 596, 258, 341, 632, 591, 406, 2320, 459, 1080, 819, 372, 2072, 774, 276, 508, 629, 920, 721, 794, 470, 400, 293, 602, 628, 634, 998, 390, 1500, 537, 713, 1227, 748, 1124, 1439, 559, 431, 1311, 681, 573, 579, 1303, 600, 1120, 502, 195, 763, 1686, 1341, 1242, 552, 910, 332, 919, 237] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=7 tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 564, 337, 908, 488, 398, 430, 928, 376, 2005, 879, 916, 584, 649, 1980, 1010, 388, 905, 589, 842, 384, 756, 548, 353, 465, 423, 169, 630, 691, 628, 1425, 1151, 819, 1227, 308, 774, 1284, 632, 509, 2387, 902, 571, 979, 1073, 690, 1103, 793, 208, 755, 752, 1123, 652, 555, 659, 557, 677, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=7 tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 593, 281, 982, 251, 359, 423, 666, 246, 1812, 1708, 1189, 657, 964, 1995, 883, 396, 588, 833, 684, 422, 698, 1193, 509, 372, 740, 254, 1564, 875, 551, 1148, 1091, 1286, 729, 507, 870, 1470, 303, 602, 1064, 575, 647, 1176, 1100, 968, 746, 491, 283, 391, 864, 1232, 572, 478, 388, 348, 1138, 446] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=7 tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 703, 346, 420, 597, 225, 356, 622, 89, 1855, 1108, 835, 1011, 706, 2416, 1055, 546, 904, 681, 660, 494, 740, 1182, 348, 368, 491, 152, 558, 651, 561, 1504, 1424, 1375, 1017, 408, 567, 1439, 242, 506, 1745, 865, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 1097, 1196, 531, 339, 871, 248, 1343, 346] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=7 tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 1038, 352, 804, 546, 300, 519, 745, 199, 1776, 1088, 850, 468, 937, 2037, 688, 272, 868, 588, 626, 308, 550, 546, 424, 352, 493, 263, 832, 849, 634, 1441, 1127, 944, 790, 411, 1065, 1572, 315, 547, 2203, 587, 776, 1576, 798, 682, 810, 442, 199, 699, 1081, 1146, 541, 440, 720, 254, 885, 475] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=7 rerouted_tokens_per_expert_current_ep_rank=[580, 1093, 283, 204, 523, 1072, 1152, 586, 154, 107, 0, 0, 0, 0, 0, 0, 439, 380, 783, 409, 362, 609, 930, 291, 2015, 0, 0, 0, 0, 0, 0, 0, 1258, 1124, 954, 532, 682, 0, 672, 345, 0, 0, 0, 0, 0, 0, 0, 0, 1103, 946, 863, 512, 651, 803, 465, 281, 1322, 22, 6, 0, 0, 0, 0, 0, 463, 180, 1162, 623, 746, 854, 1523, 1064, 0, 0, 0, 0, 0, 0, 0, 0, 970, 769, 1103, 1115, 356, 504, 924, 597, 0, 0, 0, 0, 0, 0, 0, 0, 761, 698, 1115, 644, 917, 490, 169, 496, 429, 0, 0, 0, 0, 0, 0, 0, 972, 1516, 616, 440, 915, 278, 677, 347, 206, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5754, 6218, 5567, 6974, 6615, 6338, 5719, 5967] avg=6144.0000 max=6974 max_rank=3 min=5567 min_rank=2 imbalance_ratio=1.135091 max_load_reduction=42178 max_load_reduction_pct=0.858114 moved_to_echo=24639 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=7 rerouted_tokens_per_expert_current_ep_rank=[692, 938, 208, 188, 306, 1407, 1138, 674, 145, 107, 0, 0, 0, 0, 0, 0, 593, 281, 982, 251, 359, 423, 666, 246, 1990, 0, 0, 0, 0, 0, 0, 0, 1255, 1708, 1189, 657, 964, 0, 883, 396, 0, 0, 0, 0, 0, 0, 0, 0, 588, 833, 684, 422, 698, 1193, 509, 372, 626, 22, 5, 0, 0, 0, 0, 0, 740, 254, 1564, 875, 551, 808, 1091, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 729, 507, 870, 1470, 303, 602, 438, 575, 0, 0, 0, 0, 0, 0, 0, 0, 647, 1176, 1100, 968, 746, 491, 283, 391, 428, 0, 0, 0, 0, 0, 0, 0, 864, 1232, 572, 478, 388, 348, 1138, 446, 195, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5803, 5791, 7052, 5952, 7169, 5494, 6230, 5661] avg=6144.0000 max=7169 max_rank=4 min=5494 min_rank=5 imbalance_ratio=1.166829 max_load_reduction=41983 max_load_reduction_pct=0.854146 moved_to_echo=24554 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=7 rerouted_tokens_per_expert_current_ep_rank=[1050, 617, 373, 182, 250, 1514, 1301, 459, 190, 137, 0, 0, 0, 0, 0, 0, 789, 510, 596, 258, 341, 632, 591, 406, 2066, 0, 0, 0, 0, 0, 0, 0, 1606, 459, 1080, 819, 372, 0, 774, 276, 0, 0, 0, 0, 0, 0, 0, 0, 508, 629, 920, 721, 794, 470, 400, 293, 771, 29, 6, 0, 0, 0, 0, 0, 602, 628, 634, 998, 390, 1055, 537, 713, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 748, 1124, 1439, 559, 431, 540, 681, 0, 0, 0, 0, 0, 0, 0, 0, 573, 579, 1303, 600, 1120, 502, 195, 763, 548, 0, 0, 0, 0, 0, 0, 0, 1686, 1341, 1242, 552, 910, 332, 919, 237, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6189, 5386, 5541, 5557, 6749, 6183, 7474] avg=6144.0000 max=7474 max_rank=7 min=5386 min_rank=2 imbalance_ratio=1.216471 max_load_reduction=41678 max_load_reduction_pct=0.847941 moved_to_echo=25963 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=7 rerouted_tokens_per_expert_current_ep_rank=[925, 1172, 315, 191, 363, 1410, 1226, 678, 141, 128, 0, 0, 0, 0, 0, 0, 819, 270, 704, 338, 210, 330, 911, 179, 2040, 0, 0, 0, 0, 0, 0, 0, 1494, 1205, 1014, 598, 1151, 0, 799, 961, 0, 0, 0, 0, 0, 0, 0, 0, 1097, 1194, 675, 377, 620, 803, 403, 538, 1059, 27, 6, 0, 0, 0, 0, 0, 422, 371, 655, 531, 662, 783, 1105, 996, 0, 0, 0, 0, 0, 0, 0, 0, 726, 320, 751, 1064, 452, 577, 740, 1119, 0, 0, 0, 0, 0, 0, 0, 0, 625, 869, 1149, 691, 625, 580, 143, 472, 511, 0, 0, 0, 0, 0, 0, 0, 769, 1066, 713, 562, 804, 247, 906, 586, 189, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6549, 5801, 7222, 6799, 5525, 5749, 5665, 5842] avg=6144.0000 max=7222 max_rank=2 min=5525 min_rank=4 imbalance_ratio=1.175456 max_load_reduction=41930 max_load_reduction_pct=0.853068 moved_to_echo=22781 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=7 rerouted_tokens_per_expert_current_ep_rank=[656, 935, 197, 294, 402, 1145, 1729, 511, 184, 123, 0, 0, 0, 0, 0, 0, 564, 337, 908, 488, 398, 430, 928, 376, 1975, 0, 0, 0, 0, 0, 0, 0, 1376, 879, 916, 584, 649, 0, 1010, 388, 0, 0, 0, 0, 0, 0, 0, 0, 905, 589, 842, 384, 756, 548, 353, 465, 1409, 28, 5, 0, 0, 0, 0, 0, 423, 169, 630, 691, 628, 996, 1151, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1227, 308, 774, 1284, 632, 509, 978, 902, 0, 0, 0, 0, 0, 0, 0, 0, 571, 979, 1073, 690, 1103, 793, 208, 755, 478, 0, 0, 0, 0, 0, 0, 0, 752, 1123, 652, 555, 659, 557, 677, 495, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6176, 6404, 5802, 6284, 5507, 6614, 6650, 5715] avg=6144.0000 max=6650 max_rank=6 min=5507 min_rank=4 imbalance_ratio=1.082357 max_load_reduction=42502 max_load_reduction_pct=0.864705 moved_to_echo=24486 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=7 rerouted_tokens_per_expert_current_ep_rank=[400, 1053, 307, 232, 210, 1564, 1330, 689, 163, 125, 0, 0, 0, 0, 0, 0, 1051, 376, 592, 314, 225, 458, 801, 242, 2390, 0, 0, 0, 0, 0, 0, 0, 1457, 938, 1035, 790, 643, 0, 754, 429, 0, 0, 0, 0, 0, 0, 0, 0, 728, 671, 843, 407, 667, 602, 348, 347, 933, 26, 8, 0, 0, 0, 0, 0, 663, 373, 934, 571, 660, 906, 1223, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 1013, 678, 913, 1168, 437, 562, 652, 830, 0, 0, 0, 0, 0, 0, 0, 0, 613, 1397, 888, 848, 479, 433, 136, 625, 498, 0, 0, 0, 0, 0, 0, 0, 1114, 1428, 1102, 633, 637, 224, 783, 341, 218, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6073, 6449, 6046, 5580, 6354, 6253, 5917, 6480] avg=6144.0000 max=6480 max_rank=7 min=5580 min_rank=3 imbalance_ratio=1.054688 max_load_reduction=42672 max_load_reduction_pct=0.868164 moved_to_echo=25004 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=7 rerouted_tokens_per_expert_current_ep_rank=[597, 1117, 236, 295, 403, 1185, 1227, 401, 191, 110, 0, 0, 0, 0, 0, 0, 703, 346, 420, 597, 225, 356, 622, 89, 2409, 0, 0, 0, 0, 0, 0, 0, 1284, 1108, 835, 1011, 706, 0, 1055, 546, 0, 0, 0, 0, 0, 0, 0, 0, 904, 681, 660, 494, 740, 1182, 348, 368, 1027, 23, 7, 0, 0, 0, 0, 0, 491, 152, 558, 651, 561, 1058, 1424, 1375, 0, 0, 0, 0, 0, 0, 0, 0, 1017, 408, 567, 1439, 242, 506, 718, 865, 0, 0, 0, 0, 0, 0, 0, 0, 633, 1134, 1057, 1060, 1145, 304, 195, 420, 438, 0, 0, 0, 0, 0, 0, 0, 1097, 1196, 531, 339, 871, 248, 1343, 346, 255, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5762, 5767, 6545, 6434, 6270, 5762, 6386, 6226] avg=6144.0000 max=6545 max_rank=2 min=5762 min_rank=0 imbalance_ratio=1.065267 max_load_reduction=42607 max_load_reduction_pct=0.866842 moved_to_echo=24644 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=7 rerouted_tokens_per_expert_current_ep_rank=[749, 974, 208, 221, 417, 931, 2067, 1107, 183, 105, 0, 0, 0, 0, 0, 0, 1038, 352, 804, 546, 300, 519, 745, 199, 2030, 0, 0, 0, 0, 0, 0, 0, 1229, 1088, 850, 468, 937, 0, 688, 272, 0, 0, 0, 0, 0, 0, 0, 0, 868, 588, 626, 308, 550, 546, 424, 352, 1297, 22, 7, 0, 0, 0, 0, 0, 493, 263, 832, 849, 634, 1013, 1127, 944, 0, 0, 0, 0, 0, 0, 0, 0, 790, 411, 1065, 1572, 315, 547, 906, 587, 0, 0, 0, 0, 0, 0, 0, 0, 776, 1576, 798, 682, 810, 442, 199, 699, 420, 0, 0, 0, 0, 0, 0, 0, 1081, 1146, 541, 440, 720, 254, 885, 475, 245, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6962, 6533, 5532, 5588, 6155, 6193, 6402, 5787] avg=6144.0000 max=6962 max_rank=0 min=5532 min_rank=2 imbalance_ratio=1.133138 max_load_reduction=42190 max_load_reduction_pct=0.858358 moved_to_echo=24537 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 2944, 5888, 3328, 2432, 3840, 6272, 2048, 17024, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 4224, 7168, 10624, 3328, 4352, 6016, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[8448, 10112, 6016, 4096, 6016, 2560, 7424, 3328, 1920, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5248, 8448, 8576, 6272, 7040, 4096, 1536, 4736, 3840, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5760, 7936, 2176, 1920, 2944, 10240, 11264, 5120, 1408, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6784, 6144, 6144, 3712, 5504, 6272, 3328, 3072, 8448, 256, 128, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[11008, 8576, 7936, 5504, 6144, 0, 6656, 3712, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 2432, 7040, 5888, 4864, 7552, 9216, 8320, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=8 tokens_per_expert_current_ep_rank=[1766, 312, 2267, 678, 375, 855, 570, 1400, 934, 907, 866, 1000, 635, 502, 304, 475, 1118, 1017, 1326, 669, 395, 1724, 1169, 1781, 357, 329, 532, 403, 1024, 750, 1324, 246, 832, 774, 1296, 244, 1218, 933, 625, 671, 591, 1666, 164, 259, 390, 491, 217, 619, 620, 207, 490, 452, 795, 1013, 300, 257, 465, 533, 190, 472, 855, 857, 2105, 541] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=8 tokens_per_expert_current_ep_rank=[1367, 308, 2513, 401, 358, 825, 696, 1439, 617, 1555, 716, 685, 965, 383, 885, 364, 969, 709, 1547, 1058, 236, 1359, 981, 1583, 428, 295, 782, 382, 728, 489, 1840, 269, 589, 765, 893, 653, 787, 881, 383, 655, 234, 1399, 182, 177, 649, 375, 467, 402, 743, 48, 541, 921, 1296, 923, 303, 260, 310, 1681, 595, 263, 618, 1252, 1402, 773] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=8 tokens_per_expert_current_ep_rank=[1443, 508, 1849, 1029, 336, 989, 435, 1446, 1154, 790, 1174, 729, 948, 421, 423, 508, 820, 384, 1066, 748, 501, 913, 953, 1201, 353, 889, 878, 323, 748, 407, 1245, 232, 1093, 601, 938, 315, 844, 1035, 706, 1027, 498, 1317, 541, 300, 495, 444, 570, 577, 624, 444, 526, 1030, 904, 911, 381, 298, 713, 1469, 214, 682, 966, 1333, 1056, 457] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=8 tokens_per_expert_current_ep_rank=[2113, 473, 2218, 326, 308, 806, 308, 1348, 1223, 927, 829, 1225, 740, 490, 305, 380, 1182, 1161, 1190, 707, 363, 2461, 1617, 1949, 439, 321, 822, 256, 778, 372, 1216, 283, 604, 386, 1087, 390, 697, 481, 626, 435, 407, 928, 231, 165, 518, 278, 401, 864, 737, 223, 550, 468, 1563, 311, 360, 323, 840, 767, 389, 571, 1453, 1065, 1475, 423] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=8 tokens_per_expert_current_ep_rank=[1293, 645, 2066, 712, 215, 1181, 327, 1662, 1071, 693, 885, 1589, 654, 357, 787, 444, 734, 872, 1380, 457, 686, 987, 1108, 1085, 495, 229, 836, 197, 922, 501, 1249, 120, 441, 846, 742, 319, 783, 687, 613, 694, 528, 1624, 642, 200, 671, 829, 355, 411, 652, 221, 477, 1153, 1444, 758, 369, 219, 1139, 625, 357, 877, 1180, 1141, 1353, 363] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=8 tokens_per_expert_current_ep_rank=[1287, 477, 2069, 608, 371, 831, 1031, 1390, 1083, 701, 811, 1088, 1048, 340, 618, 775, 988, 317, 934, 949, 560, 1097, 1076, 1219, 451, 387, 578, 429, 639, 452, 1140, 229, 537, 684, 877, 240, 476, 793, 590, 904, 228, 857, 650, 248, 938, 445, 496, 689, 701, 484, 887, 1163, 1280, 1351, 510, 581, 737, 980, 266, 775, 482, 1454, 1273, 603] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=8 tokens_per_expert_current_ep_rank=[1784, 543, 2085, 588, 263, 737, 772, 1366, 831, 845, 707, 1143, 907, 480, 629, 483, 1588, 337, 1316, 347, 767, 1795, 985, 1678, 482, 184, 652, 416, 776, 403, 1284, 151, 447, 636, 1260, 162, 614, 691, 588, 743, 896, 1139, 271, 146, 660, 132, 300, 988, 936, 210, 385, 903, 1366, 849, 520, 349, 592, 635, 445, 591, 748, 1326, 1896, 404] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=8 tokens_per_expert_current_ep_rank=[2070, 507, 1893, 766, 336, 922, 837, 1694, 1198, 987, 670, 1019, 704, 469, 748, 221, 828, 678, 1081, 663, 687, 827, 1195, 1202, 216, 278, 334, 458, 788, 384, 1993, 280, 547, 522, 864, 309, 528, 836, 624, 1009, 712, 1850, 388, 149, 588, 462, 271, 503, 1146, 402, 791, 1008, 1349, 695, 727, 259, 482, 657, 494, 899, 743, 739, 1246, 420] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1367, 308, 108, 401, 358, 825, 696, 1439, 0, 0, 0, 0, 0, 0, 0, 0, 617, 1555, 716, 685, 965, 383, 885, 364, 12, 0, 0, 0, 0, 0, 0, 0, 969, 709, 1547, 1058, 236, 820, 981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 428, 295, 782, 382, 728, 489, 1840, 269, 1125, 469, 0, 0, 0, 0, 0, 0, 589, 765, 893, 653, 787, 881, 383, 655, 409, 199, 0, 0, 0, 0, 0, 0, 234, 1399, 182, 177, 649, 375, 467, 402, 1936, 0, 0, 0, 0, 0, 0, 0, 743, 48, 541, 921, 1296, 923, 303, 260, 458, 340, 0, 0, 0, 0, 0, 0, 310, 1681, 595, 263, 618, 1252, 981, 773, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5502, 6182, 6320, 6807, 6214, 5821, 5833, 6473] avg=6144.0000 max=6807 max_rank=3 min=5502 min_rank=0 imbalance_ratio=1.107910 max_load_reduction=42345 max_load_reduction_pct=0.861511 moved_to_echo=24341 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1766, 312, 97, 678, 375, 855, 570, 1400, 0, 0, 0, 0, 0, 0, 0, 0, 934, 907, 866, 1000, 635, 502, 304, 475, 19, 0, 0, 0, 0, 0, 0, 0, 1118, 1017, 1326, 669, 395, 1041, 1169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 329, 532, 403, 1024, 750, 1324, 246, 1264, 423, 0, 0, 0, 0, 0, 0, 832, 774, 1296, 244, 1218, 933, 625, 671, 614, 252, 0, 0, 0, 0, 0, 0, 591, 1666, 164, 259, 390, 491, 217, 619, 1747, 0, 0, 0, 0, 0, 0, 0, 620, 207, 490, 452, 795, 1013, 300, 257, 517, 431, 0, 0, 0, 0, 0, 0, 465, 533, 190, 472, 855, 857, 1472, 541, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6053, 5642, 6735, 6652, 7459, 6144, 5082, 5385] avg=6144.0000 max=7459 max_rank=4 min=5082 min_rank=6 imbalance_ratio=1.214030 max_load_reduction=41693 max_load_reduction_pct=0.848246 moved_to_echo=24070 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2113, 473, 95, 326, 308, 806, 308, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 1223, 927, 829, 1225, 740, 490, 305, 380, 13, 0, 0, 0, 0, 0, 0, 0, 1182, 1161, 1190, 707, 363, 1485, 1617, 0, 0, 0, 0, 0, 0, 0, 0, 0, 439, 321, 822, 256, 778, 372, 1216, 283, 1385, 414, 0, 0, 0, 0, 0, 0, 604, 386, 1087, 390, 697, 481, 626, 435, 430, 360, 0, 0, 0, 0, 0, 0, 407, 928, 231, 165, 518, 278, 401, 864, 1709, 0, 0, 0, 0, 0, 0, 0, 737, 223, 550, 468, 1563, 311, 360, 323, 564, 616, 0, 0, 0, 0, 0, 0, 840, 767, 389, 571, 1453, 1065, 1032, 423, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5777, 6132, 7705, 6286, 5496, 5501, 5715, 6540] avg=6144.0000 max=7705 max_rank=2 min=5496 min_rank=4 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1784, 543, 90, 588, 263, 737, 772, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 831, 845, 707, 1143, 907, 480, 629, 483, 17, 0, 0, 0, 0, 0, 0, 0, 1588, 337, 1316, 347, 767, 1084, 985, 0, 0, 0, 0, 0, 0, 0, 0, 0, 482, 184, 652, 416, 776, 403, 1284, 151, 1192, 389, 0, 0, 0, 0, 0, 0, 447, 636, 1260, 162, 614, 691, 588, 743, 553, 262, 0, 0, 0, 0, 0, 0, 896, 1139, 271, 146, 660, 132, 300, 988, 1606, 0, 0, 0, 0, 0, 0, 0, 936, 210, 385, 903, 1366, 849, 520, 349, 486, 449, 0, 0, 0, 0, 0, 0, 592, 635, 445, 591, 748, 1326, 1326, 404, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6143, 6042, 6424, 5929, 5956, 6138, 6453, 6067] avg=6144.0000 max=6453 max_rank=6 min=5929 min_rank=3 imbalance_ratio=1.050293 max_load_reduction=42699 max_load_reduction_pct=0.868713 moved_to_echo=24614 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=8 rerouted_tokens_per_expert_current_ep_rank=[2070, 507, 82, 766, 336, 922, 837, 1694, 0, 0, 0, 0, 0, 0, 0, 0, 1198, 987, 670, 1019, 704, 469, 748, 221, 11, 0, 0, 0, 0, 0, 0, 0, 828, 678, 1081, 663, 687, 499, 1195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 278, 334, 458, 788, 384, 1993, 280, 853, 353, 0, 0, 0, 0, 0, 0, 547, 522, 864, 309, 528, 836, 624, 1009, 363, 121, 0, 0, 0, 0, 0, 0, 712, 1850, 388, 149, 588, 462, 271, 503, 1458, 0, 0, 0, 0, 0, 0, 0, 1146, 402, 791, 1008, 1349, 695, 727, 259, 349, 207, 0, 0, 0, 0, 0, 0, 482, 657, 494, 899, 743, 739, 872, 420, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7214, 6027, 5631, 5937, 5723, 6381, 6933, 5306] avg=6144.0000 max=7214 max_rank=0 min=5306 min_rank=7 imbalance_ratio=1.174154 max_load_reduction=41938 max_load_reduction_pct=0.853231 moved_to_echo=24343 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1287, 477, 89, 608, 371, 831, 1031, 1390, 0, 0, 0, 0, 0, 0, 0, 0, 1083, 701, 811, 1088, 1048, 340, 618, 775, 11, 0, 0, 0, 0, 0, 0, 0, 988, 317, 934, 949, 560, 663, 1076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 451, 387, 578, 429, 639, 452, 1140, 229, 865, 386, 0, 0, 0, 0, 0, 0, 537, 684, 877, 240, 476, 793, 590, 904, 371, 160, 0, 0, 0, 0, 0, 0, 228, 857, 650, 248, 938, 445, 496, 689, 1594, 0, 0, 0, 0, 0, 0, 0, 701, 484, 887, 1163, 1280, 1351, 510, 581, 354, 274, 0, 0, 0, 0, 0, 0, 737, 980, 266, 775, 482, 1454, 891, 603, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6084, 6475, 5487, 5556, 5632, 6145, 7585, 6188] avg=6144.0000 max=7585 max_rank=6 min=5487 min_rank=2 imbalance_ratio=1.234538 max_load_reduction=41567 max_load_reduction_pct=0.845683 moved_to_echo=25550 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1443, 508, 70, 1029, 336, 989, 435, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 1154, 790, 1174, 729, 948, 421, 423, 508, 13, 0, 0, 0, 0, 0, 0, 0, 820, 384, 1066, 748, 501, 545, 953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 353, 889, 878, 323, 748, 407, 1245, 232, 853, 351, 0, 0, 0, 0, 0, 0, 1093, 601, 938, 315, 844, 1035, 706, 1027, 312, 137, 0, 0, 0, 0, 0, 0, 498, 1317, 541, 300, 495, 444, 570, 577, 1428, 0, 0, 0, 0, 0, 0, 0, 624, 444, 526, 1030, 904, 911, 381, 298, 348, 231, 0, 0, 0, 0, 0, 0, 713, 1469, 214, 682, 966, 1333, 731, 457, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6256, 6160, 5017, 6279, 7008, 6170, 5697, 6565] avg=6144.0000 max=7008 max_rank=4 min=5017 min_rank=2 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=25440 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=8 rerouted_tokens_per_expert_current_ep_rank=[1293, 645, 88, 712, 215, 1181, 327, 1662, 0, 0, 0, 0, 0, 0, 0, 0, 1071, 693, 885, 1589, 654, 357, 787, 444, 12, 0, 0, 0, 0, 0, 0, 0, 734, 872, 1380, 457, 686, 596, 1108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 229, 836, 197, 922, 501, 1249, 120, 771, 386, 0, 0, 0, 0, 0, 0, 441, 846, 742, 319, 783, 687, 613, 694, 395, 144, 0, 0, 0, 0, 0, 0, 528, 1624, 642, 200, 671, 829, 355, 411, 1592, 0, 0, 0, 0, 0, 0, 0, 652, 221, 477, 1153, 1444, 758, 369, 219, 314, 247, 0, 0, 0, 0, 0, 0, 1139, 625, 357, 877, 1180, 1141, 946, 363, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6123, 6492, 5833, 5706, 5664, 6852, 5854, 6628] avg=6144.0000 max=6852 max_rank=5 min=5664 min_rank=4 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=24998 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[13184, 3840, 768, 5120, 2688, 7168, 4992, 11776, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8320, 5504, 9856, 5632, 4224, 6784, 9088, 0, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[6272, 2304, 4736, 7168, 10112, 6912, 3584, 2560, 3456, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8192, 7424, 6784, 8576, 6656, 3456, 4736, 3712, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 10880, 3072, 1664, 4992, 3456, 3200, 5120, 13184, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5376, 7424, 3072, 5248, 7168, 9216, 8320, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3328, 2944, 5504, 2944, 6528, 3840, 11392, 1920, 8320, 3200, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5120, 5248, 8064, 2688, 6016, 6400, 4864, 6144, 3456, 1664, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49408 slack_slots=896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=9 tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 481, 134, 819, 294, 441, 1046, 1618, 495, 404, 399, 442, 331, 1870, 1101, 430, 320, 433, 680, 1210, 1101, 1020, 1752, 1380, 504, 1374, 1160, 1510, 341, 707, 1785, 589, 1139, 407, 1038, 471, 1003, 505, 932, 997, 292, 320, 887, 60, 990, 461, 504, 1690, 1032, 692, 274, 1412, 804, 855, 722, 379, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=9 tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1042, 98, 1201, 328, 662, 1650, 2076, 369, 358, 554, 597, 187, 1052, 880, 415, 1084, 548, 470, 849, 582, 1029, 1219, 1112, 694, 657, 227, 801, 294, 864, 1657, 461, 1018, 240, 970, 539, 1754, 307, 757, 813, 578, 280, 814, 46, 1364, 879, 574, 1803, 448, 793, 277, 1196, 838, 454, 795, 639, 471] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=9 tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 498, 254, 1524, 467, 361, 991, 1131, 402, 213, 371, 246, 426, 1559, 1064, 132, 391, 447, 739, 1903, 1377, 1066, 1043, 1004, 407, 1714, 408, 1085, 275, 1551, 2124, 522, 1895, 190, 346, 351, 2286, 645, 1039, 705, 476, 233, 487, 42, 1279, 1174, 737, 1266, 827, 410, 303, 1575, 505, 359, 443, 155, 625] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=9 tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 770, 94, 1607, 227, 604, 963, 1573, 389, 324, 328, 384, 388, 775, 1589, 354, 608, 429, 670, 1699, 1397, 942, 1088, 1130, 517, 1002, 496, 1189, 189, 1179, 1869, 1942, 1052, 284, 1460, 427, 1170, 619, 422, 1019, 591, 621, 652, 142, 447, 806, 860, 802, 636, 765, 177, 1513, 617, 1169, 985, 160, 424] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=9 tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 506, 237, 1550, 455, 883, 1661, 1463, 214, 264, 254, 414, 472, 1224, 1334, 308, 511, 520, 181, 1550, 1189, 1113, 1644, 537, 320, 1792, 238, 2205, 365, 695, 1613, 346, 1599, 286, 1386, 505, 917, 577, 519, 1155, 511, 186, 598, 69, 1258, 1818, 851, 2126, 123, 623, 127, 1785, 616, 981, 793, 151, 484] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=9 tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 647, 138, 1518, 398, 686, 1296, 1110, 867, 356, 275, 217, 651, 1187, 1034, 278, 1026, 767, 453, 1150, 1101, 1081, 827, 1750, 411, 831, 613, 295, 94, 805, 1507, 745, 1476, 246, 1553, 540, 1119, 751, 343, 619, 377, 255, 904, 112, 1188, 1356, 1446, 1464, 513, 671, 154, 2209, 623, 940, 741, 292, 865] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=9 tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 370, 227, 876, 346, 1388, 1182, 1316, 438, 674, 343, 757, 490, 907, 969, 362, 682, 498, 629, 1042, 857, 961, 1905, 1568, 513, 921, 654, 923, 255, 697, 1624, 650, 918, 676, 1775, 596, 647, 570, 399, 907, 384, 236, 881, 126, 1083, 1251, 447, 2059, 255, 538, 191, 1957, 418, 632, 996, 362, 874] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=9 tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 429, 186, 1436, 485, 496, 1115, 1051, 930, 263, 286, 269, 500, 1111, 1156, 454, 721, 399, 606, 1092, 1243, 945, 1421, 885, 452, 1561, 496, 1007, 312, 879, 2096, 360, 1250, 404, 657, 541, 1415, 392, 612, 652, 955, 265, 1083, 60, 1106, 955, 814, 1337, 634, 613, 663, 1766, 706, 668, 763, 170, 365] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=9 rerouted_tokens_per_expert_current_ep_rank=[772, 842, 652, 540, 443, 316, 122, 792, 1315, 0, 0, 0, 0, 0, 0, 0, 481, 134, 819, 294, 441, 1046, 1425, 495, 0, 0, 0, 0, 0, 0, 0, 0, 404, 399, 442, 331, 1870, 1101, 430, 320, 1060, 403, 0, 0, 0, 0, 0, 0, 433, 680, 1210, 1101, 1020, 0, 1380, 504, 0, 0, 0, 0, 0, 0, 0, 0, 1374, 1160, 1510, 341, 707, 67, 589, 1139, 0, 0, 0, 0, 0, 0, 0, 0, 407, 1038, 471, 1003, 505, 932, 997, 292, 193, 117, 51, 0, 0, 0, 0, 0, 320, 887, 60, 990, 461, 504, 1573, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 692, 274, 1412, 804, 855, 722, 379, 636, 641, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5794, 5135, 6760, 6328, 6887, 6006, 5827, 6415] avg=6144.0000 max=6887 max_rank=4 min=5135 min_rank=1 imbalance_ratio=1.120931 max_load_reduction=42265 max_load_reduction_pct=0.859884 moved_to_echo=25135 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=9 rerouted_tokens_per_expert_current_ep_rank=[605, 481, 390, 725, 378, 166, 288, 1584, 1377, 0, 0, 0, 0, 0, 0, 0, 770, 94, 1607, 227, 604, 963, 1385, 389, 0, 0, 0, 0, 0, 0, 0, 0, 324, 328, 384, 388, 775, 1589, 354, 608, 660, 422, 0, 0, 0, 0, 0, 0, 429, 670, 1699, 1397, 942, 0, 1130, 517, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 496, 1189, 189, 1179, 70, 1942, 1052, 0, 0, 0, 0, 0, 0, 0, 0, 284, 1460, 427, 1170, 619, 422, 1019, 591, 188, 55, 30, 0, 0, 0, 0, 0, 621, 652, 142, 447, 806, 860, 747, 636, 0, 0, 0, 0, 0, 0, 0, 0, 765, 177, 1513, 617, 1169, 985, 160, 424, 398, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5994, 6039, 5832, 6784, 7119, 6265, 4911, 6208] avg=6144.0000 max=7119 max_rank=4 min=4911 min_rank=6 imbalance_ratio=1.158691 max_load_reduction=42033 max_load_reduction_pct=0.855164 moved_to_echo=24503 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=9 rerouted_tokens_per_expert_current_ep_rank=[920, 781, 289, 765, 321, 402, 227, 576, 1110, 0, 0, 0, 0, 0, 0, 0, 647, 138, 1518, 398, 686, 1296, 978, 867, 0, 0, 0, 0, 0, 0, 0, 0, 356, 275, 217, 651, 1187, 1034, 278, 1026, 500, 340, 0, 0, 0, 0, 0, 0, 767, 453, 1150, 1101, 1081, 0, 1750, 411, 0, 0, 0, 0, 0, 0, 0, 0, 831, 613, 295, 94, 805, 57, 745, 1476, 0, 0, 0, 0, 0, 0, 0, 0, 246, 1553, 540, 1119, 751, 343, 619, 377, 132, 101, 23, 0, 0, 0, 0, 0, 255, 904, 112, 1188, 1356, 1446, 1363, 513, 0, 0, 0, 0, 0, 0, 0, 0, 671, 154, 2209, 623, 940, 741, 292, 865, 304, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5391, 6528, 5864, 6713, 4916, 5804, 7137, 6799] avg=6144.0000 max=7137 max_rank=6 min=4916 min_rank=4 imbalance_ratio=1.161621 max_load_reduction=42015 max_load_reduction_pct=0.854797 moved_to_echo=24656 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=9 rerouted_tokens_per_expert_current_ep_rank=[1156, 1475, 723, 613, 292, 334, 868, 1027, 1221, 0, 0, 0, 0, 0, 0, 0, 1042, 98, 1201, 328, 662, 1650, 1828, 369, 0, 0, 0, 0, 0, 0, 0, 0, 358, 554, 597, 187, 1052, 880, 415, 1084, 739, 374, 0, 0, 0, 0, 0, 0, 548, 470, 849, 582, 1029, 0, 1112, 694, 0, 0, 0, 0, 0, 0, 0, 0, 657, 227, 801, 294, 864, 62, 461, 1018, 0, 0, 0, 0, 0, 0, 0, 0, 240, 970, 539, 1754, 307, 757, 813, 578, 248, 125, 34, 0, 0, 0, 0, 0, 280, 814, 46, 1364, 879, 574, 1678, 448, 0, 0, 0, 0, 0, 0, 0, 0, 793, 277, 1196, 838, 454, 795, 639, 471, 446, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7709, 7178, 6240, 5284, 4384, 6365, 6083, 5909] avg=6144.0000 max=7709 max_rank=0 min=4384 min_rank=4 imbalance_ratio=1.254720 max_load_reduction=41443 max_load_reduction_pct=0.843160 moved_to_echo=22741 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=9 rerouted_tokens_per_expert_current_ep_rank=[546, 712, 164, 553, 232, 241, 138, 464, 1188, 0, 0, 0, 0, 0, 0, 0, 506, 237, 1550, 455, 883, 1661, 1288, 214, 0, 0, 0, 0, 0, 0, 0, 0, 264, 254, 414, 472, 1224, 1334, 308, 511, 995, 364, 0, 0, 0, 0, 0, 0, 520, 181, 1550, 1189, 1113, 0, 537, 320, 0, 0, 0, 0, 0, 0, 0, 0, 1792, 238, 2205, 365, 695, 61, 346, 1599, 0, 0, 0, 0, 0, 0, 0, 0, 286, 1386, 505, 917, 577, 519, 1155, 511, 175, 147, 46, 0, 0, 0, 0, 0, 186, 598, 69, 1258, 1818, 851, 1979, 123, 0, 0, 0, 0, 0, 0, 0, 0, 623, 127, 1785, 616, 981, 793, 151, 484, 603, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4238, 6794, 6140, 5410, 7301, 6224, 6882, 6163] avg=6144.0000 max=7301 max_rank=4 min=4238 min_rank=0 imbalance_ratio=1.188314 max_load_reduction=41851 max_load_reduction_pct=0.851461 moved_to_echo=26570 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=9 rerouted_tokens_per_expert_current_ep_rank=[731, 1181, 310, 489, 429, 270, 303, 1237, 1199, 0, 0, 0, 0, 0, 0, 0, 370, 227, 876, 346, 1388, 1182, 1155, 438, 0, 0, 0, 0, 0, 0, 0, 0, 674, 343, 757, 490, 907, 969, 362, 682, 1155, 371, 0, 0, 0, 0, 0, 0, 498, 629, 1042, 857, 961, 0, 1568, 513, 0, 0, 0, 0, 0, 0, 0, 0, 921, 654, 923, 255, 697, 54, 650, 918, 0, 0, 0, 0, 0, 0, 0, 0, 676, 1775, 596, 647, 570, 399, 907, 384, 161, 146, 53, 0, 0, 0, 0, 0, 236, 881, 126, 1083, 1251, 447, 1913, 255, 0, 0, 0, 0, 0, 0, 0, 0, 538, 191, 1957, 418, 632, 996, 362, 874, 697, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6149, 5982, 6710, 6068, 5072, 6314, 6192, 6665] avg=6144.0000 max=6710 max_rank=2 min=5072 min_rank=4 imbalance_ratio=1.092122 max_load_reduction=42442 max_load_reduction_pct=0.863485 moved_to_echo=24243 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=9 rerouted_tokens_per_expert_current_ep_rank=[675, 927, 862, 696, 374, 340, 95, 1135, 1565, 0, 0, 0, 0, 0, 0, 0, 498, 254, 1524, 467, 361, 991, 996, 402, 0, 0, 0, 0, 0, 0, 0, 0, 213, 371, 246, 426, 1559, 1064, 132, 391, 631, 480, 0, 0, 0, 0, 0, 0, 447, 739, 1903, 1377, 1066, 0, 1004, 407, 0, 0, 0, 0, 0, 0, 0, 0, 1714, 408, 1085, 275, 1551, 79, 522, 1895, 0, 0, 0, 0, 0, 0, 0, 0, 190, 346, 351, 2286, 645, 1039, 705, 476, 135, 87, 30, 0, 0, 0, 0, 0, 233, 487, 42, 1279, 1174, 737, 1179, 827, 0, 0, 0, 0, 0, 0, 0, 0, 410, 303, 1575, 505, 359, 443, 155, 625, 382, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6669, 5493, 5513, 6943, 7529, 6290, 5958, 4757] avg=6144.0000 max=7529 max_rank=4 min=4757 min_rank=7 imbalance_ratio=1.225423 max_load_reduction=41623 max_load_reduction_pct=0.846822 moved_to_echo=24534 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=9 rerouted_tokens_per_expert_current_ep_rank=[827, 1647, 286, 658, 389, 409, 162, 1286, 1544, 0, 0, 0, 0, 0, 0, 0, 429, 186, 1436, 485, 496, 1115, 926, 930, 0, 0, 0, 0, 0, 0, 0, 0, 263, 286, 269, 500, 1111, 1156, 454, 721, 860, 473, 0, 0, 0, 0, 0, 0, 399, 606, 1092, 1243, 945, 0, 885, 452, 0, 0, 0, 0, 0, 0, 0, 0, 1561, 496, 1007, 312, 879, 79, 360, 1250, 0, 0, 0, 0, 0, 0, 0, 0, 404, 657, 541, 1415, 392, 612, 652, 955, 125, 92, 39, 0, 0, 0, 0, 0, 265, 1083, 60, 1106, 955, 814, 1245, 634, 0, 0, 0, 0, 0, 0, 0, 0, 613, 663, 1766, 706, 668, 763, 170, 365, 522, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7208, 6003, 6093, 5622, 5944, 5884, 6162, 6236] avg=6144.0000 max=7208 max_rank=0 min=5622 min_rank=3 imbalance_ratio=1.173177 max_load_reduction=41944 max_load_reduction_pct=0.853353 moved_to_echo=24226 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6272, 8064, 3712, 5120, 2944, 2560, 2304, 8192, 10624, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 9216, 4096, 10368, 4480, 5120, 6912, 4224, 1408, 896, 384, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 2176, 13440, 5248, 6144, 6272, 2432, 4864, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 2816, 3328, 3456, 9728, 9216, 2816, 5376, 6656, 3328, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 1408, 10624, 3072, 5632, 9984, 9984, 4224, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2432, 6400, 768, 8832, 8704, 6272, 11776, 4480, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4096, 4480, 10496, 8960, 8192, 0, 9472, 3840, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[9856, 4352, 9088, 2176, 7424, 640, 5632, 10368, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=10 tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1600, 185, 920, 578, 434, 548, 133, 275, 648, 850, 789, 744, 819, 1731, 495, 251, 1864, 239, 923, 552, 398, 323, 61, 542, 1503, 1502, 336, 2610, 325, 557, 342, 310, 1025, 703, 670, 645, 487, 2412, 1866, 1101, 612, 201, 450, 711, 380, 168, 855, 508, 1428, 1713, 367, 557, 488, 355, 462, 689, 638, 1270, 1055, 2215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=10 tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 2229, 229, 1576, 350, 384, 789, 393, 407, 655, 900, 603, 600, 633, 2057, 151, 387, 1533, 612, 1324, 582, 271, 249, 326, 613, 616, 1358, 255, 1482, 342, 557, 688, 492, 1242, 584, 701, 684, 676, 1255, 1179, 1392, 380, 557, 382, 983, 460, 611, 571, 486, 1383, 1165, 573, 879, 676, 449, 548, 469, 798, 191, 701, 2108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=10 tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 329, 111, 756, 232, 272, 416, 350, 325, 1134, 655, 543, 682, 734, 1948, 59, 174, 1518, 379, 1668, 737, 670, 346, 147, 715, 970, 1509, 261, 1276, 465, 365, 632, 267, 790, 446, 382, 691, 1089, 1931, 1403, 1196, 611, 803, 432, 834, 460, 587, 596, 792, 1281, 1667, 228, 831, 1028, 1120, 637, 325, 471, 584, 1006, 1834] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=10 tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 591, 160, 857, 494, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1419, 1328, 81, 380, 1260, 320, 941, 924, 330, 467, 319, 340, 958, 751, 412, 1160, 227, 334, 486, 585, 836, 508, 199, 1751, 286, 2098, 866, 847, 439, 742, 356, 1126, 359, 773, 1251, 610, 1063, 1455, 245, 819, 1358, 900, 1073, 146, 392, 153, 635, 1949] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=10 tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1498, 138, 1088, 585, 498, 519, 389, 340, 477, 223, 675, 1372, 868, 2369, 107, 333, 1915, 390, 1895, 851, 345, 342, 286, 678, 1320, 925, 281, 1943, 500, 540, 659, 308, 811, 798, 319, 381, 991, 808, 1341, 840, 452, 409, 761, 528, 415, 677, 531, 939, 1483, 1562, 416, 798, 765, 775, 675, 215, 534, 730, 1015, 1827] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=10 tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1997, 154, 1551, 578, 593, 592, 330, 333, 1053, 400, 585, 1053, 941, 1567, 113, 345, 1776, 551, 1380, 476, 626, 335, 210, 614, 575, 1237, 278, 1030, 206, 558, 683, 556, 350, 655, 448, 582, 927, 1255, 1755, 731, 504, 510, 682, 786, 252, 562, 1285, 915, 1019, 1648, 595, 518, 1763, 641, 369, 263, 582, 141, 1371, 1499] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=10 tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 2199, 151, 1751, 507, 313, 667, 245, 436, 193, 599, 835, 1163, 715, 2081, 145, 735, 1741, 423, 891, 556, 477, 379, 138, 980, 876, 1307, 343, 1775, 246, 691, 424, 321, 681, 896, 648, 328, 912, 1329, 1013, 1256, 509, 273, 709, 760, 399, 581, 601, 370, 1580, 1734, 200, 856, 821, 370, 666, 228, 292, 843, 808, 1706] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=10 tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1427, 103, 994, 738, 310, 795, 257, 333, 663, 139, 1761, 692, 1292, 1795, 84, 470, 2363, 272, 1048, 692, 440, 307, 95, 1026, 494, 901, 311, 1374, 352, 746, 860, 178, 855, 744, 1219, 588, 1186, 1271, 1094, 1444, 1034, 513, 620, 583, 337, 186, 210, 1145, 1782, 1386, 657, 539, 414, 454, 448, 309, 645, 729, 999, 1923] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=10 rerouted_tokens_per_expert_current_ep_rank=[429, 368, 665, 272, 1126, 185, 920, 578, 0, 0, 0, 0, 0, 0, 0, 0, 434, 548, 133, 275, 648, 850, 789, 744, 1209, 474, 104, 0, 0, 0, 0, 0, 819, 562, 495, 251, 1864, 239, 923, 552, 0, 0, 0, 0, 0, 0, 0, 0, 398, 323, 61, 542, 1503, 1502, 336, 2610, 298, 247, 0, 0, 0, 0, 0, 0, 325, 557, 342, 310, 1025, 703, 670, 645, 1169, 417, 0, 0, 0, 0, 0, 0, 487, 786, 1866, 1101, 612, 201, 450, 711, 0, 0, 0, 0, 0, 0, 0, 0, 380, 168, 855, 508, 1428, 1415, 367, 557, 0, 0, 0, 0, 0, 0, 0, 0, 488, 355, 462, 689, 638, 1270, 1055, 1864, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4543, 6208, 5705, 7820, 6163, 6214, 5678, 6821] avg=6144.0000 max=7820 max_rank=3 min=4543 min_rank=0 imbalance_ratio=1.272786 max_load_reduction=41332 max_load_reduction_pct=0.840902 moved_to_echo=24876 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1888, 662, 1094, 788, 416, 160, 857, 494, 0, 0, 0, 0, 0, 0, 0, 0, 224, 582, 270, 335, 2024, 127, 1770, 1029, 1052, 175, 91, 0, 0, 0, 0, 0, 1419, 431, 81, 380, 1260, 320, 941, 924, 0, 0, 0, 0, 0, 0, 0, 0, 330, 467, 319, 340, 958, 751, 412, 1160, 253, 217, 0, 0, 0, 0, 0, 0, 227, 334, 486, 585, 836, 508, 199, 1751, 897, 363, 0, 0, 0, 0, 0, 0, 286, 683, 866, 847, 439, 742, 356, 1126, 0, 0, 0, 0, 0, 0, 0, 0, 359, 773, 1251, 610, 1063, 1202, 245, 819, 0, 0, 0, 0, 0, 0, 0, 0, 1358, 900, 1073, 146, 392, 153, 635, 1641, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6359, 7679, 5756, 5207, 6186, 5345, 6322, 6298] avg=6144.0000 max=7679 max_rank=1 min=5207 min_rank=3 imbalance_ratio=1.249837 max_load_reduction=41473 max_load_reduction_pct=0.843770 moved_to_echo=24151 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1140, 620, 1237, 429, 1568, 229, 1576, 350, 0, 0, 0, 0, 0, 0, 0, 0, 384, 789, 393, 407, 655, 900, 603, 600, 629, 661, 99, 0, 0, 0, 0, 0, 633, 668, 151, 387, 1533, 612, 1324, 582, 0, 0, 0, 0, 0, 0, 0, 0, 271, 249, 326, 613, 616, 1358, 255, 1482, 202, 235, 0, 0, 0, 0, 0, 0, 342, 557, 688, 492, 1242, 584, 701, 684, 1389, 217, 0, 0, 0, 0, 0, 0, 676, 409, 1179, 1392, 380, 557, 382, 983, 0, 0, 0, 0, 0, 0, 0, 0, 460, 611, 571, 486, 1383, 963, 573, 879, 0, 0, 0, 0, 0, 0, 0, 0, 676, 449, 548, 469, 798, 191, 701, 1774, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7149, 6120, 5890, 5607, 6896, 5958, 5926, 5606] avg=6144.0000 max=7149 max_rank=0 min=5606 min_rank=7 imbalance_ratio=1.163574 max_load_reduction=42003 max_load_reduction_pct=0.854553 moved_to_echo=24386 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=10 rerouted_tokens_per_expert_current_ep_rank=[922, 742, 2241, 547, 232, 111, 756, 232, 0, 0, 0, 0, 0, 0, 0, 0, 272, 416, 350, 325, 1134, 655, 543, 682, 968, 97, 86, 0, 0, 0, 0, 0, 734, 632, 59, 174, 1518, 379, 1668, 737, 0, 0, 0, 0, 0, 0, 0, 0, 670, 346, 147, 715, 970, 1509, 261, 1276, 290, 204, 0, 0, 0, 0, 0, 0, 465, 365, 632, 267, 790, 446, 382, 691, 1316, 334, 0, 0, 0, 0, 0, 0, 1089, 629, 1403, 1196, 611, 803, 432, 834, 0, 0, 0, 0, 0, 0, 0, 0, 460, 587, 596, 792, 1281, 1377, 228, 831, 0, 0, 0, 0, 0, 0, 0, 0, 1028, 1120, 637, 325, 471, 584, 1006, 1544, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5783, 5528, 5901, 6388, 5688, 6997, 6152, 6715] avg=6144.0000 max=6997 max_rank=5 min=5528 min_rank=1 imbalance_ratio=1.138835 max_load_reduction=42155 max_load_reduction_pct=0.857646 moved_to_echo=25552 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=10 rerouted_tokens_per_expert_current_ep_rank=[1385, 398, 1036, 449, 1402, 154, 1551, 578, 0, 0, 0, 0, 0, 0, 0, 0, 593, 592, 330, 333, 1053, 400, 585, 1053, 632, 595, 74, 0, 0, 0, 0, 0, 941, 505, 113, 345, 1776, 551, 1380, 476, 0, 0, 0, 0, 0, 0, 0, 0, 626, 335, 210, 614, 575, 1237, 278, 1030, 291, 171, 0, 0, 0, 0, 0, 0, 206, 558, 683, 556, 350, 655, 448, 582, 1062, 220, 0, 0, 0, 0, 0, 0, 927, 403, 1755, 731, 504, 510, 682, 786, 0, 0, 0, 0, 0, 0, 0, 0, 252, 562, 1285, 915, 1019, 1357, 595, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1763, 641, 369, 263, 582, 141, 1371, 1254, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6953, 6240, 6087, 5367, 5320, 6298, 6503, 6384] avg=6144.0000 max=6953 max_rank=0 min=5320 min_rank=4 imbalance_ratio=1.131673 max_load_reduction=42199 max_load_reduction_pct=0.858541 moved_to_echo=24505 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=10 rerouted_tokens_per_expert_current_ep_rank=[685, 410, 1262, 342, 1054, 138, 1088, 585, 0, 0, 0, 0, 0, 0, 0, 0, 498, 519, 389, 340, 477, 223, 675, 1372, 405, 444, 86, 0, 0, 0, 0, 0, 868, 769, 107, 333, 1915, 390, 1895, 851, 0, 0, 0, 0, 0, 0, 0, 0, 345, 342, 286, 678, 1320, 925, 281, 1943, 271, 203, 0, 0, 0, 0, 0, 0, 500, 540, 659, 308, 811, 798, 319, 381, 1600, 139, 0, 0, 0, 0, 0, 0, 991, 264, 1341, 840, 452, 409, 761, 528, 0, 0, 0, 0, 0, 0, 0, 0, 415, 677, 531, 939, 1483, 1291, 416, 798, 0, 0, 0, 0, 0, 0, 0, 0, 765, 775, 675, 215, 534, 730, 1015, 1538, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5564, 5428, 7128, 6594, 6055, 5586, 6550, 6247] avg=6144.0000 max=7128 max_rank=2 min=5428 min_rank=1 imbalance_ratio=1.160156 max_load_reduction=42024 max_load_reduction_pct=0.854980 moved_to_echo=24438 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=10 rerouted_tokens_per_expert_current_ep_rank=[486, 612, 1942, 440, 1547, 151, 1751, 507, 0, 0, 0, 0, 0, 0, 0, 0, 313, 667, 245, 436, 193, 599, 835, 1163, 666, 652, 80, 0, 0, 0, 0, 0, 715, 675, 145, 735, 1741, 423, 891, 556, 0, 0, 0, 0, 0, 0, 0, 0, 477, 379, 138, 980, 876, 1307, 343, 1775, 301, 190, 0, 0, 0, 0, 0, 0, 246, 691, 424, 321, 681, 896, 648, 328, 1406, 230, 0, 0, 0, 0, 0, 0, 912, 433, 1013, 1256, 509, 273, 709, 760, 0, 0, 0, 0, 0, 0, 0, 0, 399, 581, 601, 370, 1580, 1433, 200, 856, 0, 0, 0, 0, 0, 0, 0, 0, 821, 370, 666, 228, 292, 843, 808, 1436, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7436, 5849, 5881, 6766, 5871, 5865, 6020, 5464] avg=6144.0000 max=7436 max_rank=0 min=5464 min_rank=7 imbalance_ratio=1.210286 max_load_reduction=41716 max_load_reduction_pct=0.848714 moved_to_echo=23220 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=10 rerouted_tokens_per_expert_current_ep_rank=[241, 513, 882, 890, 1004, 103, 994, 738, 0, 0, 0, 0, 0, 0, 0, 0, 310, 795, 257, 333, 663, 139, 1761, 692, 637, 423, 90, 0, 0, 0, 0, 0, 1292, 583, 84, 470, 2363, 272, 1048, 692, 0, 0, 0, 0, 0, 0, 0, 0, 440, 307, 95, 1026, 494, 901, 311, 1374, 241, 214, 0, 0, 0, 0, 0, 0, 352, 746, 860, 178, 855, 744, 1219, 588, 1212, 219, 0, 0, 0, 0, 0, 0, 1186, 415, 1094, 1444, 1034, 513, 620, 583, 0, 0, 0, 0, 0, 0, 0, 0, 337, 186, 210, 1145, 1782, 1145, 657, 539, 0, 0, 0, 0, 0, 0, 0, 0, 414, 454, 448, 309, 645, 729, 999, 1619, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5365, 6100, 6804, 5403, 6973, 6889, 6001, 5617] avg=6144.0000 max=6973 max_rank=4 min=5365 min_rank=0 imbalance_ratio=1.134928 max_load_reduction=42179 max_load_reduction_pct=0.858134 moved_to_echo=25480 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6656, 4096, 10624, 8832, 4608, 4096, 4480, 6400, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7424, 5120, 4992, 2688, 4352, 4736, 7680, 12672, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3072, 4992, 2432, 2816, 6912, 3968, 7680, 7424, 6272, 3584, 768, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[7296, 4352, 10368, 4224, 8448, 1280, 9600, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 4224, 6016, 5888, 11136, 10240, 3328, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[3584, 2816, 1664, 5632, 7424, 9600, 2560, 12672, 2176, 1792, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7424, 4864, 1280, 3200, 14080, 3200, 10112, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2688, 4352, 4864, 3072, 6656, 5376, 4608, 5760, 10112, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=11 tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 1043, 115, 749, 269, 387, 194, 674, 1403, 584, 248, 104, 1047, 156, 876, 570, 178, 981, 1958, 905, 343, 878, 503, 1346, 460, 623, 699, 491, 2904, 606, 1050, 1378, 1721, 1175, 1044, 836, 876, 739, 292, 595, 202, 189, 701, 1309, 677, 223, 789, 1111, 1151, 500, 289, 683, 1508, 1181, 494, 831, 1438, 441, 387] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=11 tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 2072, 250, 1362, 600, 1165, 265, 399, 967, 410, 292, 122, 661, 373, 880, 491, 376, 277, 1361, 145, 529, 484, 1184, 802, 248, 635, 469, 464, 1935, 840, 916, 1359, 1250, 1101, 799, 1183, 319, 629, 324, 789, 254, 408, 1293, 364, 152, 303, 951, 489, 1716, 710, 1047, 372, 2199, 1091, 569, 1160, 1611, 435, 566] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=11 tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 2013, 219, 954, 428, 274, 340, 561, 1260, 307, 250, 273, 1875, 440, 614, 1106, 106, 408, 1659, 358, 216, 1215, 501, 2079, 143, 918, 809, 1110, 1693, 604, 1537, 1133, 1050, 1619, 208, 1549, 526, 609, 131, 564, 742, 158, 635, 1044, 563, 331, 1163, 217, 572, 1150, 531, 440, 454, 931, 1057, 852, 2078, 482, 313] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=11 tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2386, 265, 853, 266, 899, 475, 649, 637, 444, 207, 168, 893, 298, 819, 537, 268, 871, 1404, 565, 1298, 984, 575, 861, 229, 492, 713, 620, 2519, 517, 452, 1349, 1053, 1112, 1383, 1123, 753, 503, 276, 518, 104, 54, 587, 1249, 373, 264, 278, 450, 2125, 882, 1069, 287, 1843, 587, 562, 783, 1568, 250, 620] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=11 tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1365, 179, 1367, 343, 769, 296, 576, 343, 273, 498, 179, 1047, 256, 1382, 726, 61, 306, 2382, 626, 304, 782, 503, 2296, 278, 561, 548, 797, 1186, 1092, 1714, 1425, 1541, 690, 385, 1254, 858, 509, 377, 329, 80, 271, 485, 847, 253, 219, 725, 1051, 331, 1694, 399, 468, 1332, 1197, 876, 620, 1208, 893, 239] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=11 tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1637, 127, 1114, 307, 103, 308, 606, 303, 409, 230, 58, 509, 285, 806, 977, 88, 352, 2377, 668, 297, 799, 424, 1748, 836, 523, 619, 1698, 2460, 1371, 757, 1312, 1356, 1833, 790, 1706, 486, 314, 207, 347, 256, 76, 660, 988, 336, 349, 921, 503, 1251, 735, 319, 901, 613, 1265, 667, 681, 1196, 745, 591] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=11 tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1249, 224, 975, 700, 281, 372, 1467, 340, 975, 862, 85, 580, 148, 1427, 678, 291, 439, 460, 539, 719, 615, 148, 1336, 689, 199, 534, 1387, 2797, 1061, 1053, 828, 1122, 1347, 773, 1123, 545, 390, 280, 639, 198, 82, 521, 1072, 216, 286, 878, 644, 2082, 577, 312, 908, 1329, 506, 387, 1494, 2767, 395, 660] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=11 tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 1127, 198, 1949, 741, 982, 255, 756, 815, 415, 489, 69, 1931, 504, 618, 390, 62, 1021, 1136, 567, 549, 1148, 620, 2138, 41, 552, 467, 1238, 2279, 874, 1403, 785, 1554, 1330, 271, 1151, 233, 591, 201, 294, 662, 178, 566, 655, 342, 342, 1430, 442, 267, 502, 505, 440, 1127, 924, 1471, 862, 1392, 122, 922] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=11 rerouted_tokens_per_expert_current_ep_rank=[861, 1844, 559, 789, 443, 239, 1784, 250, 0, 0, 0, 0, 0, 0, 0, 0, 1362, 600, 1165, 265, 399, 967, 410, 292, 846, 321, 0, 0, 0, 0, 0, 0, 122, 661, 373, 880, 491, 376, 277, 1361, 395, 243, 0, 0, 0, 0, 0, 0, 145, 529, 484, 1184, 802, 248, 635, 469, 518, 0, 0, 0, 0, 0, 0, 0, 464, 0, 840, 916, 1359, 9, 1101, 799, 0, 0, 0, 0, 0, 0, 0, 0, 1183, 319, 629, 324, 789, 254, 408, 1293, 1614, 0, 0, 0, 0, 0, 0, 0, 364, 152, 303, 951, 489, 1716, 710, 1047, 223, 288, 0, 0, 0, 0, 0, 0, 372, 2199, 1091, 569, 1160, 627, 435, 566, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6769, 6627, 5179, 5014, 5488, 6813, 6243, 7019] avg=6144.0000 max=7019 max_rank=7 min=5014 min_rank=3 imbalance_ratio=1.142415 max_load_reduction=42133 max_load_reduction_pct=0.857198 moved_to_echo=25563 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=11 rerouted_tokens_per_expert_current_ep_rank=[353, 1886, 1265, 608, 468, 981, 1176, 179, 0, 0, 0, 0, 0, 0, 0, 0, 1367, 343, 769, 296, 576, 343, 273, 498, 1043, 196, 0, 0, 0, 0, 0, 0, 179, 1047, 256, 1382, 726, 61, 306, 2382, 487, 182, 0, 0, 0, 0, 0, 0, 626, 304, 782, 503, 2296, 278, 561, 548, 389, 0, 0, 0, 0, 0, 0, 0, 797, 0, 1092, 1714, 1425, 11, 690, 385, 0, 0, 0, 0, 0, 0, 0, 0, 1254, 858, 509, 377, 329, 80, 271, 485, 990, 0, 0, 0, 0, 0, 0, 0, 847, 253, 219, 725, 1051, 331, 1694, 399, 167, 189, 0, 0, 0, 0, 0, 0, 468, 1332, 1197, 876, 620, 470, 893, 239, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6916, 5704, 7008, 6287, 6114, 5153, 5875, 6095] avg=6144.0000 max=7008 max_rank=2 min=5153 min_rank=5 imbalance_ratio=1.140625 max_load_reduction=42144 max_load_reduction_pct=0.857422 moved_to_echo=23237 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=11 rerouted_tokens_per_expert_current_ep_rank=[767, 860, 1299, 305, 507, 310, 898, 115, 0, 0, 0, 0, 0, 0, 0, 0, 749, 269, 387, 194, 674, 1403, 584, 248, 1165, 482, 0, 0, 0, 0, 0, 0, 104, 1047, 156, 876, 570, 178, 981, 1958, 544, 217, 0, 0, 0, 0, 0, 0, 905, 343, 878, 503, 1346, 460, 623, 699, 463, 0, 0, 0, 0, 0, 0, 0, 491, 0, 606, 1050, 1378, 12, 1175, 1044, 0, 0, 0, 0, 0, 0, 0, 0, 836, 876, 739, 292, 595, 202, 189, 701, 2422, 0, 0, 0, 0, 0, 0, 0, 1309, 677, 223, 789, 1111, 1151, 500, 289, 199, 145, 0, 0, 0, 0, 0, 0, 683, 1508, 1181, 494, 831, 559, 441, 387, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5061, 6155, 6631, 6220, 5756, 6852, 6393, 6084] avg=6144.0000 max=6852 max_rank=5 min=5061 min_rank=0 imbalance_ratio=1.115234 max_load_reduction=42300 max_load_reduction_pct=0.860596 moved_to_echo=25085 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=11 rerouted_tokens_per_expert_current_ep_rank=[294, 1224, 1065, 1503, 298, 599, 2050, 265, 0, 0, 0, 0, 0, 0, 0, 0, 853, 266, 899, 475, 649, 637, 444, 207, 716, 419, 0, 0, 0, 0, 0, 0, 168, 893, 298, 819, 537, 268, 871, 1404, 336, 242, 0, 0, 0, 0, 0, 0, 565, 1298, 984, 575, 861, 229, 492, 713, 507, 0, 0, 0, 0, 0, 0, 0, 620, 0, 517, 452, 1349, 1, 1112, 1383, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 753, 503, 276, 518, 104, 54, 587, 2100, 0, 0, 0, 0, 0, 0, 0, 1249, 373, 264, 278, 450, 2125, 882, 1069, 223, 336, 0, 0, 0, 0, 0, 0, 287, 1843, 587, 562, 783, 596, 250, 620, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7298, 5565, 5836, 6224, 5434, 6018, 7249, 5528] avg=6144.0000 max=7298 max_rank=0 min=5434 min_rank=4 imbalance_ratio=1.187826 max_load_reduction=41854 max_load_reduction_pct=0.851522 moved_to_echo=24229 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=11 rerouted_tokens_per_expert_current_ep_rank=[431, 1229, 1182, 417, 470, 51, 1733, 219, 0, 0, 0, 0, 0, 0, 0, 0, 954, 428, 274, 340, 561, 1260, 307, 250, 711, 281, 0, 0, 0, 0, 0, 0, 273, 1875, 440, 614, 1106, 106, 408, 1659, 331, 314, 0, 0, 0, 0, 0, 0, 358, 216, 1215, 501, 2079, 143, 918, 809, 669, 0, 0, 0, 0, 0, 0, 0, 1110, 0, 604, 1537, 1133, 8, 1619, 208, 0, 0, 0, 0, 0, 0, 0, 0, 1549, 526, 609, 131, 564, 742, 158, 635, 1412, 0, 0, 0, 0, 0, 0, 0, 1044, 563, 331, 1163, 217, 572, 1150, 531, 288, 280, 0, 0, 0, 0, 0, 0, 440, 454, 931, 1057, 852, 807, 482, 313, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5732, 5366, 7126, 6908, 6219, 6326, 6139, 5336] avg=6144.0000 max=7126 max_rank=2 min=5336 min_rank=7 imbalance_ratio=1.159831 max_load_reduction=42026 max_load_reduction_pct=0.855021 moved_to_echo=24020 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=11 rerouted_tokens_per_expert_current_ep_rank=[162, 1684, 883, 836, 323, 273, 1076, 224, 0, 0, 0, 0, 0, 0, 0, 0, 975, 700, 281, 372, 1467, 340, 975, 862, 759, 465, 0, 0, 0, 0, 0, 0, 85, 580, 148, 1427, 678, 291, 439, 460, 354, 418, 0, 0, 0, 0, 0, 0, 539, 719, 615, 148, 1336, 689, 199, 534, 891, 0, 0, 0, 0, 0, 0, 0, 1387, 0, 1061, 1053, 828, 9, 1347, 773, 0, 0, 0, 0, 0, 0, 0, 0, 1123, 545, 390, 280, 639, 198, 82, 521, 2332, 0, 0, 0, 0, 0, 0, 0, 1072, 216, 286, 878, 644, 2082, 577, 312, 384, 173, 0, 0, 0, 0, 0, 0, 908, 1329, 506, 387, 1494, 1074, 395, 660, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 7196, 4880, 5670, 6458, 6110, 6624, 6753] avg=6144.0000 max=7196 max_rank=1 min=4880 min_rank=2 imbalance_ratio=1.171224 max_load_reduction=41956 max_load_reduction_pct=0.853597 moved_to_echo=25945 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=11 rerouted_tokens_per_expert_current_ep_rank=[1203, 2182, 843, 415, 231, 78, 1410, 127, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 307, 103, 308, 606, 303, 409, 230, 918, 408, 0, 0, 0, 0, 0, 0, 58, 509, 285, 806, 977, 88, 352, 2377, 428, 180, 0, 0, 0, 0, 0, 0, 668, 297, 799, 424, 1748, 836, 523, 619, 385, 0, 0, 0, 0, 0, 0, 0, 1698, 0, 1371, 757, 1312, 10, 1833, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 486, 314, 207, 347, 256, 76, 660, 2052, 0, 0, 0, 0, 0, 0, 0, 988, 336, 349, 921, 503, 1251, 735, 319, 166, 227, 0, 0, 0, 0, 0, 0, 901, 613, 1265, 667, 681, 465, 745, 591, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6489, 4706, 6060, 6299, 7771, 6104, 5795, 5928] avg=6144.0000 max=7771 max_rank=4 min=4706 min_rank=1 imbalance_ratio=1.264811 max_load_reduction=41381 max_load_reduction_pct=0.841899 moved_to_echo=25598 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=11 rerouted_tokens_per_expert_current_ep_rank=[100, 1576, 1710, 283, 498, 90, 971, 198, 0, 0, 0, 0, 0, 0, 0, 0, 1949, 741, 982, 255, 756, 815, 415, 489, 1052, 379, 0, 0, 0, 0, 0, 0, 69, 1931, 504, 618, 390, 62, 1021, 1136, 491, 210, 0, 0, 0, 0, 0, 0, 567, 549, 1148, 620, 2138, 41, 552, 467, 448, 0, 0, 0, 0, 0, 0, 0, 1238, 0, 874, 1403, 785, 11, 1330, 271, 0, 0, 0, 0, 0, 0, 0, 0, 1151, 233, 591, 201, 294, 662, 178, 566, 1900, 0, 0, 0, 0, 0, 0, 0, 655, 342, 342, 1430, 442, 267, 502, 505, 193, 156, 0, 0, 0, 0, 0, 0, 440, 1127, 924, 1471, 862, 541, 122, 922, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5426, 7833, 6432, 6530, 5912, 5776, 4834, 6409] avg=6144.0000 max=7833 max_rank=1 min=4834 min_rank=6 imbalance_ratio=1.274902 max_load_reduction=41319 max_load_reduction_pct=0.840637 moved_to_echo=22931 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 3712, 4864, 2560, 5760, 6144, 3840, 3200, 7296, 3072, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 10496, 7808, 6144, 7296, 5248, 3840, 4352, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7552, 2944, 2432, 7168, 4992, 9600, 6784, 4480, 1920, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1152, 8576, 2560, 7424, 5504, 1536, 4736, 12800, 3456, 2048, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7808, 0, 7040, 8960, 9600, 128, 10240, 5760, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4224, 12544, 8832, 5248, 3328, 2688, 11136, 1664, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9984, 4608, 4352, 2176, 4096, 2560, 1536, 5504, 14848, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4480, 4352, 6912, 4480, 12672, 2944, 4608, 4864, 4352, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=12 tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 1100, 282, 393, 237, 2271, 486, 429, 385, 901, 322, 1260, 678, 635, 597, 732, 529, 379, 1046, 410, 1578, 48, 726, 405, 971, 576, 856, 640, 918, 2761, 356, 73, 1892, 499, 1403, 380, 2874, 480, 1202, 377, 502, 216, 355, 31, 655, 240, 810, 1833, 1055, 67, 1200, 619, 599, 540, 2233, 61, 123] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=12 tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 814, 457, 616, 457, 1451, 968, 1135, 840, 626, 592, 632, 691, 974, 522, 618, 657, 350, 758, 1005, 1070, 210, 712, 1070, 766, 598, 1104, 1715, 1105, 2194, 1226, 176, 2383, 895, 1546, 606, 374, 419, 2575, 111, 422, 202, 616, 608, 257, 81, 964, 557, 606, 191, 748, 1122, 260, 516, 1023, 109, 208] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=12 tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 828, 190, 334, 1201, 1740, 254, 1108, 764, 870, 1457, 424, 726, 299, 437, 989, 730, 144, 749, 1265, 1726, 440, 304, 1666, 917, 883, 852, 1383, 1371, 2384, 527, 791, 1461, 1205, 1449, 1025, 263, 350, 1288, 233, 553, 372, 538, 522, 615, 57, 1259, 370, 291, 77, 373, 1869, 447, 580, 707, 222, 75] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=12 tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 903, 513, 600, 441, 1742, 890, 1386, 2407, 507, 510, 427, 347, 509, 382, 1415, 586, 94, 571, 960, 522, 350, 181, 1138, 92, 1117, 830, 334, 931, 2328, 519, 252, 400, 516, 1341, 761, 1923, 785, 1741, 244, 612, 1147, 188, 141, 477, 770, 1261, 819, 749, 466, 1487, 950, 154, 1342, 404, 102, 94] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=12 tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 414, 374, 386, 1349, 1317, 1009, 1240, 705, 820, 1197, 768, 287, 496, 505, 742, 649, 131, 515, 1669, 1002, 213, 978, 2392, 223, 457, 698, 1180, 1064, 3093, 536, 106, 468, 906, 884, 879, 726, 823, 1874, 89, 533, 1047, 113, 67, 569, 366, 1210, 329, 661, 567, 807, 1604, 297, 879, 1145, 183, 82] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=12 tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 549, 396, 724, 1286, 1809, 405, 601, 1708, 1606, 88, 484, 326, 405, 234, 1320, 777, 834, 657, 1221, 608, 82, 422, 1876, 416, 912, 688, 1440, 1101, 3858, 614, 271, 2174, 1298, 1517, 510, 272, 1044, 915, 145, 76, 541, 225, 478, 265, 297, 954, 562, 1080, 842, 346, 405, 117, 1594, 333, 669, 112] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=12 tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 930, 291, 520, 304, 1836, 711, 986, 2805, 449, 284, 644, 538, 1043, 522, 795, 546, 141, 648, 2766, 398, 146, 206, 1429, 79, 1380, 791, 1422, 1086, 2378, 1430, 224, 1660, 1185, 1687, 1311, 426, 1014, 919, 63, 532, 933, 72, 391, 210, 483, 673, 486, 990, 306, 776, 613, 262, 1509, 389, 968, 32] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=12 tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 833, 204, 187, 1337, 1251, 667, 1062, 1224, 667, 372, 277, 388, 361, 183, 1082, 2490, 258, 951, 1253, 905, 419, 139, 955, 124, 2605, 1029, 1152, 1521, 1931, 668, 180, 514, 1786, 1535, 535, 643, 528, 1739, 437, 788, 464, 226, 160, 842, 73, 998, 568, 377, 417, 372, 1479, 134, 1105, 703, 237, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=12 rerouted_tokens_per_expert_current_ep_rank=[295, 590, 255, 1549, 1627, 377, 511, 722, 720, 216, 0, 0, 0, 0, 0, 0, 1100, 282, 393, 237, 936, 486, 429, 385, 0, 0, 0, 0, 0, 0, 0, 0, 901, 322, 1260, 678, 635, 597, 732, 529, 511, 175, 0, 0, 0, 0, 0, 0, 379, 1046, 410, 1578, 48, 726, 405, 971, 427, 0, 0, 0, 0, 0, 0, 0, 576, 856, 640, 918, 0, 356, 73, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 499, 1403, 380, 2874, 480, 475, 377, 502, 0, 0, 0, 0, 0, 0, 0, 0, 216, 355, 31, 655, 240, 810, 1833, 1055, 1819, 0, 0, 0, 0, 0, 0, 0, 67, 1200, 619, 599, 540, 2233, 61, 123, 942, 615, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6862, 4248, 6340, 5990, 4709, 6990, 7014, 6999] avg=6144.0000 max=7014 max_rank=6 min=4248 min_rank=1 imbalance_ratio=1.141602 max_load_reduction=42138 max_load_reduction_pct=0.857300 moved_to_echo=25712 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=12 rerouted_tokens_per_expert_current_ep_rank=[129, 124, 573, 2707, 362, 526, 130, 112, 573, 165, 0, 0, 0, 0, 0, 0, 549, 396, 724, 1286, 747, 405, 601, 1708, 0, 0, 0, 0, 0, 0, 0, 0, 1606, 88, 484, 326, 405, 234, 1320, 777, 389, 201, 0, 0, 0, 0, 0, 0, 834, 657, 1221, 608, 82, 422, 1876, 416, 490, 0, 0, 0, 0, 0, 0, 0, 912, 688, 1440, 1101, 0, 614, 271, 1483, 0, 0, 0, 0, 0, 0, 0, 0, 1298, 1517, 510, 272, 1044, 361, 145, 76, 0, 0, 0, 0, 0, 0, 0, 0, 541, 225, 478, 265, 297, 954, 562, 1080, 2541, 0, 0, 0, 0, 0, 0, 0, 842, 346, 405, 117, 1594, 333, 669, 112, 1317, 489, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5401, 6416, 5830, 6606, 6509, 5223, 6943, 6224] avg=6144.0000 max=6943 max_rank=6 min=5223 min_rank=5 imbalance_ratio=1.130046 max_load_reduction=42209 max_load_reduction_pct=0.858744 moved_to_echo=24899 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=12 rerouted_tokens_per_expert_current_ep_rank=[653, 526, 265, 965, 1507, 586, 348, 1644, 557, 317, 0, 0, 0, 0, 0, 0, 903, 513, 600, 441, 710, 890, 1386, 2407, 0, 0, 0, 0, 0, 0, 0, 0, 507, 510, 427, 347, 509, 382, 1415, 586, 744, 41, 0, 0, 0, 0, 0, 0, 94, 571, 960, 522, 350, 181, 1138, 92, 94, 0, 0, 0, 0, 0, 0, 0, 1117, 830, 334, 931, 0, 519, 252, 265, 0, 0, 0, 0, 0, 0, 0, 0, 516, 1341, 761, 1923, 785, 680, 244, 612, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 188, 141, 477, 770, 1261, 819, 749, 1533, 0, 0, 0, 0, 0, 0, 0, 466, 1487, 950, 154, 1342, 404, 102, 94, 795, 475, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7368, 7850, 5468, 4002, 4248, 6862, 7085, 6269] avg=6144.0000 max=7850 max_rank=1 min=4002 min_rank=3 imbalance_ratio=1.277669 max_load_reduction=41302 max_load_reduction_pct=0.840291 moved_to_echo=24464 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=12 rerouted_tokens_per_expert_current_ep_rank=[215, 340, 422, 1422, 1103, 622, 313, 1261, 396, 313, 0, 0, 0, 0, 0, 0, 833, 204, 187, 1337, 517, 667, 1062, 1224, 0, 0, 0, 0, 0, 0, 0, 0, 667, 372, 277, 388, 361, 183, 1082, 2490, 739, 47, 0, 0, 0, 0, 0, 0, 258, 951, 1253, 905, 419, 139, 955, 124, 116, 0, 0, 0, 0, 0, 0, 0, 2605, 1029, 1152, 1521, 0, 668, 180, 351, 0, 0, 0, 0, 0, 0, 0, 0, 1786, 1535, 535, 643, 528, 687, 437, 788, 0, 0, 0, 0, 0, 0, 0, 0, 464, 226, 160, 842, 73, 998, 568, 377, 1271, 0, 0, 0, 0, 0, 0, 0, 417, 372, 1479, 134, 1105, 703, 237, 119, 660, 338, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6407, 6031, 6606, 5120, 7506, 6939, 4979, 5564] avg=6144.0000 max=7506 max_rank=4 min=4979 min_rank=6 imbalance_ratio=1.221680 max_load_reduction=41646 max_load_reduction_pct=0.847290 moved_to_echo=24988 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=12 rerouted_tokens_per_expert_current_ep_rank=[276, 393, 339, 2148, 662, 516, 523, 787, 460, 464, 0, 0, 0, 0, 0, 0, 814, 457, 616, 457, 598, 968, 1135, 840, 0, 0, 0, 0, 0, 0, 0, 0, 626, 592, 632, 691, 974, 522, 618, 657, 1095, 220, 0, 0, 0, 0, 0, 0, 350, 758, 1005, 1070, 210, 712, 1070, 766, 538, 0, 0, 0, 0, 0, 0, 0, 598, 1104, 1715, 1105, 0, 1226, 176, 1625, 0, 0, 0, 0, 0, 0, 0, 0, 895, 1546, 606, 374, 419, 1016, 111, 422, 0, 0, 0, 0, 0, 0, 0, 0, 202, 616, 608, 257, 81, 964, 557, 606, 1444, 0, 0, 0, 0, 0, 0, 0, 191, 748, 1122, 260, 516, 1023, 109, 208, 750, 393, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6568, 5885, 6627, 6479, 7549, 5389, 5335, 5320] avg=6144.0000 max=7549 max_rank=4 min=5320 min_rank=7 imbalance_ratio=1.228678 max_load_reduction=41603 max_load_reduction_pct=0.846415 moved_to_echo=23593 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=12 rerouted_tokens_per_expert_current_ep_rank=[310, 212, 394, 2086, 820, 508, 113, 755, 551, 232, 0, 0, 0, 0, 0, 0, 828, 190, 334, 1201, 718, 254, 1108, 764, 0, 0, 0, 0, 0, 0, 0, 0, 870, 1457, 424, 726, 299, 437, 989, 730, 547, 135, 0, 0, 0, 0, 0, 0, 144, 749, 1265, 1726, 440, 304, 1666, 917, 329, 0, 0, 0, 0, 0, 0, 0, 883, 852, 1383, 1371, 0, 527, 791, 997, 0, 0, 0, 0, 0, 0, 0, 0, 1205, 1449, 1025, 263, 350, 509, 233, 553, 0, 0, 0, 0, 0, 0, 0, 0, 372, 538, 522, 615, 57, 1259, 370, 291, 1570, 0, 0, 0, 0, 0, 0, 0, 77, 373, 1869, 447, 580, 707, 222, 75, 814, 471, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5981, 5397, 6614, 7540, 6804, 5587, 5594, 5635] avg=6144.0000 max=7540 max_rank=3 min=5397 min_rank=1 imbalance_ratio=1.227214 max_load_reduction=41612 max_load_reduction_pct=0.846598 moved_to_echo=23620 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=12 rerouted_tokens_per_expert_current_ep_rank=[371, 66, 258, 2232, 1083, 501, 288, 730, 417, 338, 0, 0, 0, 0, 0, 0, 414, 374, 386, 1349, 544, 1009, 1240, 705, 0, 0, 0, 0, 0, 0, 0, 0, 820, 1197, 768, 287, 496, 505, 742, 649, 796, 43, 0, 0, 0, 0, 0, 0, 131, 515, 1669, 1002, 213, 978, 2392, 223, 105, 0, 0, 0, 0, 0, 0, 0, 457, 698, 1180, 1064, 0, 536, 106, 320, 0, 0, 0, 0, 0, 0, 0, 0, 906, 884, 879, 726, 823, 740, 89, 533, 0, 0, 0, 0, 0, 0, 0, 0, 1047, 113, 67, 569, 366, 1210, 329, 661, 2037, 0, 0, 0, 0, 0, 0, 0, 567, 807, 1604, 297, 879, 1145, 183, 82, 1056, 356, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6284, 6021, 6303, 7228, 4361, 5580, 6399, 6976] avg=6144.0000 max=7228 max_rank=3 min=4361 min_rank=4 imbalance_ratio=1.176432 max_load_reduction=41924 max_load_reduction_pct=0.852946 moved_to_echo=23316 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=12 rerouted_tokens_per_expert_current_ep_rank=[332, 359, 168, 1261, 384, 420, 169, 441, 582, 165, 0, 0, 0, 0, 0, 0, 930, 291, 520, 304, 757, 711, 986, 2805, 0, 0, 0, 0, 0, 0, 0, 0, 449, 284, 644, 538, 1043, 522, 795, 546, 390, 153, 0, 0, 0, 0, 0, 0, 141, 648, 2766, 398, 146, 206, 1429, 79, 374, 0, 0, 0, 0, 0, 0, 0, 1380, 791, 1422, 1086, 0, 1430, 224, 1133, 0, 0, 0, 0, 0, 0, 0, 0, 1185, 1687, 1311, 426, 1014, 364, 63, 532, 0, 0, 0, 0, 0, 0, 0, 0, 933, 72, 391, 210, 483, 673, 486, 990, 1565, 0, 0, 0, 0, 0, 0, 0, 306, 776, 613, 262, 1509, 389, 968, 32, 813, 497, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4281, 7304, 5364, 6187, 7466, 6582, 5803, 6165] avg=6144.0000 max=7466 max_rank=4 min=4281 min_rank=0 imbalance_ratio=1.215169 max_load_reduction=41686 max_load_reduction_pct=0.848104 moved_to_echo=26016 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 2688, 2688, 14464, 7552, 4096, 2432, 6528, 4352, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4992, 2432, 2432, 3968, 2432, 8192, 5632, 5888, 13824, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 2816, 3840, 6656, 5632, 5504, 8064, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8320, 11392, 6016, 7552, 5504, 4864, 1792, 4096, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6528, 4864, 4992, 4096, 4736, 3456, 7808, 7040, 5248, 1024, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 6144, 8704, 2304, 8192, 7040, 2560, 896, 7168, 3712, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8576, 6912, 9344, 9216, 0, 5888, 2176, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2432, 6016, 10624, 7936, 1920, 3712, 11008, 3712, 2560, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=13 tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 531, 601, 1742, 519, 286, 1210, 1312, 202, 122, 60, 637, 1749, 692, 507, 518, 2000, 287, 525, 534, 1818, 1021, 456, 737, 1665, 270, 372, 796, 612, 713, 379, 276, 1496, 423, 949, 840, 2407, 206, 62, 585, 1211, 1077, 303, 285, 410, 407, 913, 2174, 176, 547, 1245, 1502, 1394, 438, 1055, 539, 502] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=13 tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 660, 772, 424, 511, 217, 803, 1009, 209, 1052, 134, 479, 952, 608, 287, 875, 915, 313, 428, 335, 1706, 424, 576, 520, 1604, 455, 212, 2119, 584, 442, 388, 626, 1312, 1368, 1930, 968, 1190, 333, 426, 501, 1592, 480, 1140, 230, 381, 216, 694, 1648, 467, 379, 651, 629, 2177, 1165, 1298, 753, 396] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=13 tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 735, 1102, 134, 713, 616, 822, 827, 143, 413, 238, 1038, 3299, 413, 947, 243, 468, 451, 75, 180, 1261, 1005, 176, 379, 1545, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1069, 556, 1238, 3039, 261, 174, 13, 1345, 834, 1212, 250, 525, 891, 340, 1641, 173, 359, 863, 899, 1806, 117, 701, 1147, 1293] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=13 tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 746, 1281, 282, 954, 400, 846, 620, 263, 564, 137, 1078, 1980, 460, 556, 243, 2100, 152, 615, 193, 1143, 832, 556, 294, 1510, 353, 314, 2329, 576, 436, 243, 266, 1243, 1120, 547, 1545, 1178, 426, 463, 272, 826, 1149, 347, 127, 492, 334, 323, 2368, 229, 295, 568, 1221, 2606, 1349, 1357, 959, 339] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=13 tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 530, 665, 1044, 979, 211, 651, 867, 198, 914, 87, 1369, 1412, 1116, 320, 379, 968, 355, 712, 325, 1243, 663, 423, 119, 2479, 209, 215, 2248, 585, 201, 291, 578, 560, 767, 523, 278, 1556, 385, 712, 867, 2141, 569, 1136, 172, 332, 581, 486, 1276, 1380, 980, 832, 2030, 2129, 1624, 1152, 189, 215] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=13 tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 662, 1849, 319, 927, 623, 104, 1218, 157, 129, 290, 2455, 1752, 301, 425, 884, 4114, 148, 152, 209, 974, 223, 802, 285, 753, 253, 241, 2032, 489, 752, 197, 467, 694, 2397, 67, 510, 1010, 110, 159, 135, 1105, 907, 513, 130, 319, 487, 257, 1176, 107, 753, 870, 2832, 1797, 1869, 683, 567, 851] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=13 tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 789, 1140, 112, 861, 618, 534, 688, 209, 933, 74, 1212, 2056, 1899, 695, 639, 1231, 488, 557, 431, 1476, 826, 237, 169, 1327, 204, 148, 1776, 485, 117, 135, 259, 1025, 1009, 896, 564, 2109, 99, 228, 168, 1512, 956, 837, 169, 725, 1113, 298, 1666, 159, 483, 986, 2609, 1607, 693, 1011, 855, 108] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=13 tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 911, 620, 180, 345, 376, 1075, 625, 168, 372, 170, 601, 709, 585, 497, 130, 1111, 504, 629, 83, 1143, 999, 147, 162, 2272, 344, 858, 2112, 646, 439, 285, 203, 647, 471, 852, 833, 3059, 336, 190, 200, 1115, 620, 725, 330, 257, 698, 507, 2781, 223, 455, 1511, 2584, 550, 867, 1841, 559, 1149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=13 rerouted_tokens_per_expert_current_ep_rank=[258, 899, 615, 507, 707, 728, 153, 990, 849, 0, 0, 0, 0, 0, 0, 0, 531, 601, 1742, 519, 286, 1210, 1312, 202, 653, 149, 0, 0, 0, 0, 0, 0, 122, 60, 637, 940, 692, 507, 518, 2000, 0, 0, 0, 0, 0, 0, 0, 0, 287, 525, 534, 1818, 1021, 456, 737, 1665, 429, 29, 0, 0, 0, 0, 0, 0, 270, 372, 796, 612, 713, 379, 276, 1496, 660, 164, 0, 0, 0, 0, 0, 0, 423, 949, 840, 1437, 206, 62, 585, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 1077, 303, 285, 410, 407, 913, 2174, 176, 777, 0, 0, 0, 0, 0, 0, 0, 547, 1245, 0, 965, 438, 1055, 539, 502, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5706, 7205, 5476, 7501, 5738, 5713, 6522, 5291] avg=6144.0000 max=7501 max_rank=3 min=5291 min_rank=7 imbalance_ratio=1.220866 max_load_reduction=41651 max_load_reduction_pct=0.847392 moved_to_echo=23264 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=13 rerouted_tokens_per_expert_current_ep_rank=[112, 1010, 1762, 1143, 471, 733, 183, 775, 355, 0, 0, 0, 0, 0, 0, 0, 660, 772, 424, 511, 217, 803, 1009, 209, 274, 81, 0, 0, 0, 0, 0, 0, 1052, 134, 479, 512, 608, 287, 875, 915, 0, 0, 0, 0, 0, 0, 0, 0, 313, 428, 335, 1706, 424, 576, 520, 1604, 670, 14, 0, 0, 0, 0, 0, 0, 455, 212, 2119, 584, 442, 388, 626, 1312, 359, 81, 0, 0, 0, 0, 0, 0, 1368, 1930, 968, 711, 333, 426, 501, 1592, 0, 0, 0, 0, 0, 0, 0, 0, 480, 1140, 230, 381, 216, 694, 1648, 467, 384, 0, 0, 0, 0, 0, 0, 0, 379, 651, 0, 1507, 1165, 1298, 753, 396, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6544, 4960, 4862, 6590, 6578, 7829, 5640, 6149] avg=6144.0000 max=7829 max_rank=5 min=4862 min_rank=2 imbalance_ratio=1.274251 max_load_reduction=41323 max_load_reduction_pct=0.840719 moved_to_echo=26196 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=13 rerouted_tokens_per_expert_current_ep_rank=[270, 779, 574, 642, 822, 464, 100, 273, 1148, 0, 0, 0, 0, 0, 0, 0, 530, 665, 1044, 979, 211, 651, 867, 198, 882, 124, 0, 0, 0, 0, 0, 0, 914, 87, 1369, 751, 1116, 320, 379, 968, 0, 0, 0, 0, 0, 0, 0, 0, 355, 712, 325, 1243, 663, 423, 119, 2479, 659, 22, 0, 0, 0, 0, 0, 0, 209, 215, 2248, 585, 201, 291, 578, 560, 537, 110, 0, 0, 0, 0, 0, 0, 767, 523, 278, 917, 385, 712, 867, 2141, 0, 0, 0, 0, 0, 0, 0, 0, 569, 1136, 172, 332, 581, 486, 1276, 1380, 507, 0, 0, 0, 0, 0, 0, 0, 980, 832, 0, 1470, 1624, 1152, 189, 215, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5072, 6151, 5904, 7000, 5534, 6590, 6439, 6462] avg=6144.0000 max=7000 max_rank=3 min=5072 min_rank=0 imbalance_ratio=1.139323 max_load_reduction=42152 max_load_reduction_pct=0.857585 moved_to_echo=25025 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=13 rerouted_tokens_per_expert_current_ep_rank=[137, 369, 374, 706, 459, 1015, 137, 1464, 1602, 0, 0, 0, 0, 0, 0, 0, 662, 1849, 319, 927, 623, 104, 1218, 157, 1230, 149, 0, 0, 0, 0, 0, 0, 129, 290, 2455, 942, 301, 425, 884, 4114, 0, 0, 0, 0, 0, 0, 0, 0, 148, 152, 209, 974, 223, 802, 285, 753, 553, 12, 0, 0, 0, 0, 0, 0, 253, 241, 2032, 489, 752, 197, 467, 694, 661, 68, 0, 0, 0, 0, 0, 0, 2397, 67, 510, 604, 110, 159, 135, 1105, 0, 0, 0, 0, 0, 0, 0, 0, 907, 513, 130, 319, 487, 257, 1176, 107, 326, 0, 0, 0, 0, 0, 0, 0, 753, 870, 0, 1244, 1869, 683, 567, 851, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6263, 7238, 9540, 4111, 5854, 5087, 4222, 6837] avg=6144.0000 max=9540 max_rank=2 min=4111 min_rank=3 imbalance_ratio=1.552734 max_load_reduction=39612 max_load_reduction_pct=0.805908 moved_to_echo=22000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=13 rerouted_tokens_per_expert_current_ep_rank=[88, 572, 883, 1003, 686, 651, 181, 787, 509, 0, 0, 0, 0, 0, 0, 0, 735, 1102, 134, 713, 616, 822, 827, 143, 390, 282, 0, 0, 0, 0, 0, 0, 413, 238, 1038, 1772, 413, 947, 243, 468, 0, 0, 0, 0, 0, 0, 0, 0, 451, 75, 180, 1261, 1005, 176, 379, 1545, 556, 36, 0, 0, 0, 0, 0, 0, 144, 1040, 2091, 536, 150, 639, 326, 1406, 1245, 207, 0, 0, 0, 0, 0, 0, 1069, 556, 1238, 1815, 261, 174, 13, 1345, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1212, 250, 525, 891, 340, 1641, 173, 981, 0, 0, 0, 0, 0, 0, 0, 359, 863, 0, 1250, 117, 701, 1147, 1293, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5360, 5764, 5532, 5664, 7784, 6471, 6847, 5730] avg=6144.0000 max=7784 max_rank=4 min=5360 min_rank=0 imbalance_ratio=1.266927 max_load_reduction=41368 max_load_reduction_pct=0.841634 moved_to_echo=26832 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=13 rerouted_tokens_per_expert_current_ep_rank=[113, 642, 1083, 716, 934, 669, 437, 348, 1475, 0, 0, 0, 0, 0, 0, 0, 789, 1140, 112, 861, 618, 534, 688, 209, 1134, 175, 0, 0, 0, 0, 0, 0, 933, 74, 1212, 1105, 1899, 695, 639, 1231, 0, 0, 0, 0, 0, 0, 0, 0, 488, 557, 431, 1476, 826, 237, 169, 1327, 494, 25, 0, 0, 0, 0, 0, 0, 204, 148, 1776, 485, 117, 135, 259, 1025, 776, 143, 0, 0, 0, 0, 0, 0, 1009, 896, 564, 1260, 99, 228, 168, 1512, 0, 0, 0, 0, 0, 0, 0, 0, 956, 837, 169, 725, 1113, 298, 1666, 159, 681, 0, 0, 0, 0, 0, 0, 0, 483, 986, 0, 1113, 693, 1011, 855, 108, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6417, 6260, 7788, 6030, 5068, 5736, 6604, 5249] avg=6144.0000 max=7788 max_rank=2 min=5068 min_rank=4 imbalance_ratio=1.267578 max_load_reduction=41364 max_load_reduction_pct=0.841553 moved_to_echo=22657 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=13 rerouted_tokens_per_expert_current_ep_rank=[536, 862, 1291, 606, 111, 576, 530, 635, 690, 0, 0, 0, 0, 0, 0, 0, 746, 1281, 282, 954, 400, 846, 620, 263, 531, 169, 0, 0, 0, 0, 0, 0, 564, 137, 1078, 1064, 460, 556, 243, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 152, 615, 193, 1143, 832, 556, 294, 1510, 802, 14, 0, 0, 0, 0, 0, 0, 353, 314, 2329, 576, 436, 243, 266, 1243, 747, 80, 0, 0, 0, 0, 0, 0, 1120, 547, 1545, 704, 426, 463, 272, 826, 0, 0, 0, 0, 0, 0, 0, 0, 1149, 347, 127, 492, 334, 323, 2368, 229, 380, 0, 0, 0, 0, 0, 0, 0, 295, 568, 0, 1804, 1349, 1357, 959, 339, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5837, 6092, 6202, 6111, 6587, 5903, 5749, 6671] avg=6144.0000 max=6671 max_rank=7 min=5749 min_rank=6 imbalance_ratio=1.085775 max_load_reduction=42481 max_load_reduction_pct=0.864278 moved_to_echo=24910 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=13 rerouted_tokens_per_expert_current_ep_rank=[331, 524, 629, 222, 3034, 1020, 41, 690, 1462, 0, 0, 0, 0, 0, 0, 0, 911, 620, 180, 345, 376, 1075, 625, 168, 1122, 60, 0, 0, 0, 0, 0, 0, 372, 170, 601, 382, 585, 497, 130, 1111, 0, 0, 0, 0, 0, 0, 0, 0, 504, 629, 83, 1143, 999, 147, 162, 2272, 169, 37, 0, 0, 0, 0, 0, 0, 344, 858, 2112, 646, 439, 285, 203, 647, 267, 208, 0, 0, 0, 0, 0, 0, 471, 852, 833, 1826, 336, 190, 200, 1115, 0, 0, 0, 0, 0, 0, 0, 0, 620, 725, 330, 257, 698, 507, 2781, 223, 988, 0, 0, 0, 0, 0, 0, 0, 455, 1511, 0, 381, 867, 1841, 559, 1149, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7953, 5482, 3848, 6145, 6009, 5823, 7129, 6763] avg=6144.0000 max=7953 max_rank=0 min=3848 min_rank=2 imbalance_ratio=1.294434 max_load_reduction=41199 max_load_reduction_pct=0.838196 moved_to_echo=25724 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[5632, 8064, 4352, 5888, 3456, 6144, 7168, 1664, 6272, 1280, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4608, 1280, 8960, 7552, 6144, 4352, 3968, 12928, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1920, 5760, 7296, 5632, 7296, 5888, 1792, 6016, 8192, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4352, 7552, 0, 9856, 8192, 9216, 5632, 4864, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6656, 6272, 1792, 3456, 4736, 3840, 14848, 2944, 5120, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8704, 6400, 6784, 9344, 2176, 2432, 2816, 10880, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 3712, 2304, 10880, 6016, 3456, 2688, 13184, 4352, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 3456, 15616, 4608, 3328, 2560, 3072, 8448, 5376, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=14 tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 28, 463, 763, 1524, 977, 592, 1343, 1007, 305, 1587, 293, 736, 1731, 307, 1519, 365, 475, 88, 334, 96, 204, 1826, 345, 541, 1040, 611, 297, 652, 1968, 629, 377, 939, 2164, 177, 993, 175, 2202, 1162, 1119, 596, 1283, 397, 235, 3024, 150, 1337, 580, 567, 151, 200, 1291, 816, 749, 731, 1080, 287] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=14 tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 571, 3907, 501, 1150, 1145, 866, 1671, 1683, 1249, 931, 563, 605, 703, 403, 192, 320, 641, 80, 267, 65, 643, 575, 1059, 555, 1694, 455, 848, 797, 627, 540, 432, 2017, 1253, 706, 1160, 69, 2563, 1195, 394, 61, 404, 937, 554, 2546, 266, 1346, 636, 394, 89, 175, 807, 1054, 270, 232, 1234, 69] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=14 tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 33, 1213, 525, 716, 724, 1299, 1190, 1286, 670, 947, 163, 994, 289, 342, 392, 434, 470, 436, 212, 88, 169, 1390, 694, 429, 1117, 308, 1426, 1523, 1896, 768, 1455, 540, 1656, 356, 1450, 74, 1877, 1120, 272, 397, 410, 491, 257, 3807, 120, 2463, 225, 606, 140, 312, 587, 525, 1025, 1290, 799, 670] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=14 tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 454, 1126, 765, 1371, 734, 489, 268, 842, 1410, 486, 420, 700, 705, 1426, 961, 799, 377, 308, 246, 223, 256, 1485, 431, 321, 1230, 286, 965, 1598, 1553, 563, 1192, 818, 1734, 742, 906, 608, 1196, 884, 280, 476, 814, 732, 713, 2996, 381, 1710, 496, 553, 419, 200, 744, 1168, 507, 733, 1398, 655] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=14 tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 17, 869, 404, 1034, 552, 235, 2347, 944, 954, 1916, 1281, 484, 189, 135, 410, 277, 594, 109, 556, 8, 330, 790, 941, 125, 933, 723, 632, 896, 2678, 284, 1991, 820, 2867, 930, 833, 162, 628, 740, 269, 148, 768, 679, 363, 2478, 415, 3232, 273, 892, 736, 155, 523, 596, 2400, 906, 1039, 224] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=14 tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 69, 1184, 1011, 936, 792, 399, 1039, 869, 506, 500, 539, 1991, 247, 641, 513, 168, 323, 348, 256, 189, 79, 1227, 401, 746, 672, 794, 636, 767, 1715, 605, 336, 1602, 3184, 1085, 1577, 216, 1334, 343, 273, 67, 780, 1021, 336, 3246, 412, 1767, 204, 809, 217, 46, 770, 561, 930, 1175, 2300, 593] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=14 tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 22, 634, 388, 710, 824, 335, 1894, 1456, 1812, 991, 579, 1353, 203, 359, 285, 323, 168, 326, 224, 177, 95, 1608, 1454, 61, 572, 512, 1537, 949, 1703, 290, 1311, 2610, 2174, 942, 601, 381, 1206, 1260, 72, 160, 540, 1802, 617, 1898, 94, 1375, 189, 811, 303, 274, 835, 1382, 1348, 1001, 619, 66] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=14 tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 93, 1285, 585, 1153, 950, 126, 2342, 762, 834, 1439, 552, 1240, 193, 908, 442, 755, 615, 240, 751, 345, 113, 423, 342, 645, 784, 2030, 503, 226, 1211, 478, 2086, 355, 563, 1922, 923, 191, 687, 2456, 126, 129, 1308, 919, 786, 2039, 269, 1464, 138, 34, 593, 364, 770, 1424, 1611, 928, 650, 197] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=14 rerouted_tokens_per_expert_current_ep_rank=[712, 15, 729, 845, 353, 455, 14, 601, 1953, 1074, 0, 0, 0, 0, 0, 0, 28, 463, 763, 1524, 977, 592, 419, 1007, 0, 0, 0, 0, 0, 0, 0, 0, 305, 1587, 293, 736, 1731, 307, 1519, 365, 183, 142, 0, 0, 0, 0, 0, 0, 475, 88, 334, 96, 204, 1826, 345, 541, 1292, 894, 426, 0, 0, 0, 0, 0, 1040, 611, 297, 652, 0, 629, 377, 756, 0, 0, 0, 0, 0, 0, 0, 0, 872, 177, 993, 175, 2202, 1162, 1119, 596, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 397, 235, 1071, 150, 1337, 580, 567, 0, 0, 0, 0, 0, 0, 0, 0, 151, 200, 1291, 816, 749, 731, 1080, 287, 356, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6751, 5773, 7168, 6521, 4362, 7296, 5620, 5661] avg=6144.0000 max=7296 max_rank=5 min=4362 min_rank=4 imbalance_ratio=1.187500 max_load_reduction=41856 max_load_reduction_pct=0.851562 moved_to_echo=22939 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=14 rerouted_tokens_per_expert_current_ep_rank=[167, 24, 825, 888, 228, 499, 219, 133, 1644, 343, 0, 0, 0, 0, 0, 0, 571, 3907, 501, 1150, 1145, 866, 521, 1683, 0, 0, 0, 0, 0, 0, 0, 0, 1249, 931, 563, 605, 703, 403, 192, 320, 394, 177, 0, 0, 0, 0, 0, 0, 641, 80, 267, 65, 643, 575, 1059, 555, 748, 284, 530, 0, 0, 0, 0, 0, 1694, 455, 848, 797, 0, 540, 432, 1623, 0, 0, 0, 0, 0, 0, 0, 0, 505, 706, 1160, 69, 2563, 1195, 394, 61, 0, 0, 0, 0, 0, 0, 0, 0, 404, 937, 554, 902, 266, 1346, 636, 394, 0, 0, 0, 0, 0, 0, 0, 0, 89, 175, 807, 1054, 270, 232, 1234, 69, 443, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4970, 10344, 5537, 5447, 6389, 6653, 5439, 4373] avg=6144.0000 max=10344 max_rank=1 min=4373 min_rank=7 imbalance_ratio=1.683594 max_load_reduction=38808 max_load_reduction_pct=0.789551 moved_to_echo=22854 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=14 rerouted_tokens_per_expert_current_ep_rank=[701, 26, 799, 322, 309, 765, 31, 853, 2097, 936, 0, 0, 0, 0, 0, 0, 69, 1184, 1011, 936, 792, 399, 325, 869, 0, 0, 0, 0, 0, 0, 0, 0, 506, 500, 539, 1991, 247, 641, 513, 168, 313, 110, 0, 0, 0, 0, 0, 0, 323, 348, 256, 189, 79, 1227, 401, 746, 1901, 779, 329, 0, 0, 0, 0, 0, 672, 794, 636, 767, 0, 605, 336, 1289, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 1085, 1577, 216, 1334, 343, 273, 67, 0, 0, 0, 0, 0, 0, 0, 0, 780, 1021, 336, 1149, 412, 1767, 204, 809, 0, 0, 0, 0, 0, 0, 0, 0, 217, 46, 770, 561, 930, 1175, 2300, 593, 275, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6839, 5585, 5528, 6578, 5099, 6178, 6478, 6867] avg=6144.0000 max=6867 max_rank=7 min=5099 min_rank=4 imbalance_ratio=1.117676 max_load_reduction=42285 max_load_reduction_pct=0.860291 moved_to_echo=24622 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1909, 18, 55, 1351, 79, 353, 16, 304, 2459, 1035, 0, 0, 0, 0, 0, 0, 33, 1213, 525, 716, 724, 1299, 372, 1286, 0, 0, 0, 0, 0, 0, 0, 0, 670, 947, 163, 994, 289, 342, 392, 434, 105, 126, 0, 0, 0, 0, 0, 0, 470, 436, 212, 88, 169, 1390, 694, 429, 989, 861, 377, 0, 0, 0, 0, 0, 1117, 308, 1426, 1523, 0, 768, 1455, 435, 0, 0, 0, 0, 0, 0, 0, 0, 667, 356, 1450, 74, 1877, 1120, 272, 397, 0, 0, 0, 0, 0, 0, 0, 0, 410, 491, 257, 1348, 120, 2463, 225, 606, 0, 0, 0, 0, 0, 0, 0, 0, 140, 312, 587, 525, 1025, 1290, 799, 670, 315, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7579, 6168, 4462, 6115, 7032, 6213, 5920, 5663] avg=6144.0000 max=7579 max_rank=0 min=4462 min_rank=2 imbalance_ratio=1.233561 max_load_reduction=41573 max_load_reduction_pct=0.845805 moved_to_echo=24828 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=14 rerouted_tokens_per_expert_current_ep_rank=[1026, 38, 519, 649, 165, 290, 44, 568, 1939, 848, 0, 0, 0, 0, 0, 0, 454, 1126, 765, 1371, 734, 489, 72, 842, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 486, 420, 700, 705, 1426, 961, 799, 165, 33, 0, 0, 0, 0, 0, 0, 377, 308, 246, 223, 256, 1485, 431, 321, 1038, 705, 88, 0, 0, 0, 0, 0, 1230, 286, 965, 1598, 0, 563, 1192, 653, 0, 0, 0, 0, 0, 0, 0, 0, 696, 742, 906, 608, 1196, 884, 280, 476, 0, 0, 0, 0, 0, 0, 0, 0, 814, 732, 713, 1057, 381, 1710, 496, 553, 0, 0, 0, 0, 0, 0, 0, 0, 419, 200, 744, 1168, 507, 733, 1398, 655, 75, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6086, 5853, 7105, 5478, 6487, 5788, 6456, 5899] avg=6144.0000 max=7105 max_rank=2 min=5478 min_rank=3 imbalance_ratio=1.156413 max_load_reduction=42047 max_load_reduction_pct=0.855448 moved_to_echo=24630 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=14 rerouted_tokens_per_expert_current_ep_rank=[226, 15, 1220, 389, 185, 961, 68, 373, 1226, 929, 0, 0, 0, 0, 0, 0, 22, 634, 388, 710, 824, 335, 590, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 991, 579, 1353, 203, 359, 285, 323, 510, 201, 0, 0, 0, 0, 0, 0, 168, 326, 224, 177, 95, 1608, 1454, 61, 1298, 774, 601, 0, 0, 0, 0, 0, 572, 512, 1537, 949, 0, 290, 1311, 2100, 0, 0, 0, 0, 0, 0, 0, 0, 876, 942, 601, 381, 1206, 1260, 72, 160, 0, 0, 0, 0, 0, 0, 0, 0, 540, 1802, 617, 672, 94, 1375, 189, 811, 0, 0, 0, 0, 0, 0, 0, 0, 303, 274, 835, 1382, 1348, 1001, 619, 66, 502, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5592, 4959, 6616, 6786, 7271, 5498, 6100, 6330] avg=6144.0000 max=7271 max_rank=4 min=4959 min_rank=1 imbalance_ratio=1.183431 max_load_reduction=41881 max_load_reduction_pct=0.852071 moved_to_echo=25199 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=14 rerouted_tokens_per_expert_current_ep_rank=[604, 59, 347, 1273, 54, 494, 84, 940, 1317, 662, 0, 0, 0, 0, 0, 0, 93, 1285, 585, 1153, 950, 126, 729, 762, 0, 0, 0, 0, 0, 0, 0, 0, 834, 1439, 552, 1240, 193, 908, 442, 755, 69, 249, 0, 0, 0, 0, 0, 0, 615, 240, 751, 345, 113, 423, 342, 645, 336, 549, 743, 0, 0, 0, 0, 0, 784, 2030, 503, 226, 0, 478, 2086, 286, 0, 0, 0, 0, 0, 0, 0, 0, 227, 1922, 923, 191, 687, 2456, 126, 129, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 919, 786, 722, 269, 1464, 138, 34, 0, 0, 0, 0, 0, 0, 0, 0, 593, 364, 770, 1424, 1611, 928, 650, 197, 621, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5834, 5683, 6681, 5102, 6393, 6661, 5640, 7158] avg=6144.0000 max=7158 max_rank=7 min=5102 min_rank=3 imbalance_ratio=1.165039 max_load_reduction=41994 max_load_reduction_pct=0.854370 moved_to_echo=25852 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=14 rerouted_tokens_per_expert_current_ep_rank=[630, 20, 125, 829, 73, 410, 35, 316, 1601, 1462, 0, 0, 0, 0, 0, 0, 17, 869, 404, 1034, 552, 235, 732, 944, 0, 0, 0, 0, 0, 0, 0, 0, 954, 1916, 1281, 484, 189, 135, 410, 277, 160, 249, 0, 0, 0, 0, 0, 0, 594, 109, 556, 8, 330, 790, 941, 125, 1712, 1216, 744, 0, 0, 0, 0, 0, 933, 723, 632, 896, 0, 284, 1991, 660, 0, 0, 0, 0, 0, 0, 0, 0, 1155, 930, 833, 162, 628, 740, 269, 148, 0, 0, 0, 0, 0, 0, 0, 0, 768, 679, 363, 877, 415, 3232, 273, 892, 0, 0, 0, 0, 0, 0, 0, 0, 736, 155, 523, 596, 2400, 906, 1039, 224, 622, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5501, 4787, 6055, 7125, 6119, 4865, 7499, 7201] avg=6144.0000 max=7499 max_rank=6 min=4787 min_rank=1 imbalance_ratio=1.220540 max_load_reduction=41653 max_load_reduction_pct=0.847432 moved_to_echo=25684 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7808, 8832, 4480, 8192, 4352, 4608, 4736, 3456, 1920, 1408, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6400, 6912, 8448, 1920, 11776, 9216, 2816, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6016, 256, 4736, 6656, 1536, 4352, 512, 4096, 14336, 7296, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 10752, 4992, 8704, 6784, 4352, 3840, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2688, 1792, 6400, 7552, 8960, 7040, 9216, 2816, 3328, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[8064, 5760, 6912, 7424, 0, 4224, 9216, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6400, 7040, 3968, 7808, 2176, 14720, 2816, 4736, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3712, 2048, 2944, 1280, 1920, 9344, 5760, 3456, 9344, 6144, 3840, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49408 slack_slots=896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=15 tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 569, 766, 963, 449, 949, 844, 948, 253, 550, 930, 200, 993, 1337, 739, 206, 886, 961, 130, 329, 955, 1134, 657, 465, 167, 193, 425, 177, 356, 883, 1002, 827, 662, 3261, 438, 1563, 733, 81, 1169, 141, 548, 515, 799, 182, 475, 3122, 2531, 944, 1029, 606, 2103, 782, 406, 210, 933, 997, 1032] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=15 tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 306, 1278, 1675, 537, 501, 370, 659, 298, 1049, 912, 83, 539, 1490, 564, 415, 2021, 792, 131, 232, 1635, 1407, 533, 1068, 607, 35, 84, 529, 737, 549, 210, 446, 300, 6033, 2130, 2282, 507, 847, 341, 443, 712, 13, 1190, 395, 492, 624, 2197, 404, 525, 1757, 611, 348, 181, 121, 633, 405, 283] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=15 tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 576, 408, 1478, 396, 712, 607, 588, 1548, 2087, 332, 556, 1472, 1008, 764, 230, 116, 294, 46, 72, 973, 867, 307, 571, 439, 59, 19, 907, 114, 313, 376, 163, 421, 4767, 721, 1270, 224, 82, 1794, 933, 72, 470, 952, 307, 291, 1552, 3173, 672, 1234, 1901, 2402, 175, 95, 1269, 1054, 2030, 150] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=15 tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 358, 542, 1193, 767, 925, 1490, 1058, 362, 1383, 465, 66, 781, 717, 1415, 839, 1267, 810, 150, 186, 1855, 850, 119, 1192, 53, 10, 33, 2154, 571, 1657, 82, 518, 311, 4571, 1158, 1808, 621, 162, 474, 114, 92, 29, 859, 1551, 65, 3308, 2081, 206, 469, 897, 1201, 577, 502, 204, 447, 60, 192] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=15 tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 366, 407, 1744, 416, 460, 1119, 385, 529, 1453, 252, 303, 743, 714, 459, 1033, 1248, 519, 47, 597, 1885, 242, 164, 843, 73, 18, 287, 768, 141, 497, 141, 842, 1013, 2608, 2075, 1832, 232, 210, 683, 181, 469, 100, 1177, 871, 276, 2611, 2771, 516, 640, 1094, 2973, 1520, 485, 799, 1407, 61, 261] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=15 tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 796, 1053, 800, 509, 617, 316, 386, 512, 2639, 737, 65, 1994, 943, 2232, 106, 1750, 1002, 20, 408, 1370, 1094, 197, 562, 106, 27, 61, 911, 515, 1415, 894, 387, 500, 3757, 613, 1156, 552, 224, 1191, 76, 241, 449, 1416, 166, 411, 1477, 1493, 788, 726, 1406, 1419, 318, 93, 1175, 621, 1304, 607] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=15 tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 304, 518, 1523, 683, 351, 508, 712, 743, 949, 722, 447, 797, 555, 938, 110, 700, 226, 114, 554, 1383, 2312, 684, 157, 95, 198, 515, 1835, 626, 1110, 260, 804, 467, 3523, 1229, 983, 528, 232, 1022, 422, 111, 96, 495, 253, 558, 3082, 2430, 1549, 363, 726, 3595, 813, 424, 609, 833, 594, 222] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=15 tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 242, 475, 2557, 420, 1090, 484, 1002, 371, 689, 635, 203, 781, 1064, 2126, 374, 948, 147, 258, 708, 1148, 869, 358, 532, 248, 293, 574, 550, 459, 1202, 175, 1641, 553, 4051, 990, 1074, 502, 131, 384, 221, 907, 317, 892, 425, 176, 2885, 2176, 730, 807, 1482, 763, 527, 230, 1406, 1013, 112, 77] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=15 rerouted_tokens_per_expert_current_ep_rank=[359, 572, 140, 1022, 503, 712, 51, 327, 4325, 0, 0, 0, 0, 0, 0, 0, 306, 1278, 1675, 537, 501, 370, 659, 298, 96, 0, 0, 0, 0, 0, 0, 0, 470, 912, 83, 539, 1490, 564, 415, 2021, 0, 0, 0, 0, 0, 0, 0, 0, 792, 131, 232, 1635, 1407, 533, 1068, 607, 579, 265, 81, 0, 0, 0, 0, 0, 35, 84, 529, 737, 549, 210, 446, 300, 1672, 0, 0, 0, 0, 0, 0, 0, 1708, 2130, 2282, 507, 847, 341, 443, 712, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1190, 395, 492, 624, 260, 404, 525, 0, 0, 0, 0, 0, 0, 0, 0, 1757, 434, 348, 181, 121, 633, 405, 283, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8011, 5720, 6494, 7330, 4562, 8970, 3903, 4162] avg=6144.0000 max=8970 max_rank=5 min=3903 min_rank=6 imbalance_ratio=1.459961 max_load_reduction=40182 max_load_reduction_pct=0.817505 moved_to_echo=21597 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=15 rerouted_tokens_per_expert_current_ep_rank=[109, 538, 186, 954, 470, 25, 226, 235, 3417, 0, 0, 0, 0, 0, 0, 0, 576, 408, 1478, 396, 712, 607, 588, 1548, 379, 0, 0, 0, 0, 0, 0, 0, 934, 332, 556, 1472, 1008, 764, 230, 116, 0, 0, 0, 0, 0, 0, 0, 0, 294, 46, 72, 973, 867, 307, 571, 439, 1153, 382, 318, 0, 0, 0, 0, 0, 59, 19, 907, 114, 313, 376, 163, 421, 2415, 0, 0, 0, 0, 0, 0, 0, 1350, 721, 1270, 224, 82, 1794, 933, 72, 0, 0, 0, 0, 0, 0, 0, 0, 470, 952, 307, 291, 1552, 376, 672, 1234, 0, 0, 0, 0, 0, 0, 0, 0, 1901, 1705, 175, 95, 1269, 1054, 2030, 150, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6160, 6692, 5412, 5422, 4787, 6446, 5854, 8379] avg=6144.0000 max=8379 max_rank=7 min=4787 min_rank=4 imbalance_ratio=1.363770 max_load_reduction=40773 max_load_reduction_pct=0.829529 moved_to_echo=25466 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=15 rerouted_tokens_per_expert_current_ep_rank=[394, 631, 162, 1167, 569, 257, 103, 309, 1869, 0, 0, 0, 0, 0, 0, 0, 366, 407, 1744, 416, 460, 1119, 385, 529, 469, 0, 0, 0, 0, 0, 0, 0, 650, 252, 303, 743, 714, 459, 1033, 1248, 0, 0, 0, 0, 0, 0, 0, 0, 519, 47, 597, 1885, 242, 164, 843, 73, 803, 334, 394, 0, 0, 0, 0, 0, 18, 287, 768, 141, 497, 141, 842, 1013, 2109, 0, 0, 0, 0, 0, 0, 0, 739, 2075, 1832, 232, 210, 683, 181, 469, 0, 0, 0, 0, 0, 0, 0, 0, 100, 1177, 871, 276, 2611, 328, 516, 640, 0, 0, 0, 0, 0, 0, 0, 0, 1094, 2110, 1520, 485, 799, 1407, 61, 261, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5461, 5895, 5402, 5901, 5816, 6421, 6519, 7737] avg=6144.0000 max=7737 max_rank=7 min=5402 min_rank=2 imbalance_ratio=1.259277 max_load_reduction=41415 max_load_reduction_pct=0.842590 moved_to_echo=26493 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=15 rerouted_tokens_per_expert_current_ep_rank=[286, 63, 240, 846, 369, 455, 972, 416, 2341, 2, 0, 0, 0, 0, 0, 0, 569, 766, 963, 449, 949, 844, 948, 253, 336, 0, 0, 0, 0, 0, 0, 0, 242, 930, 200, 993, 1337, 739, 206, 886, 0, 0, 0, 0, 0, 0, 0, 0, 961, 130, 329, 955, 1134, 657, 465, 167, 308, 307, 282, 0, 0, 0, 0, 0, 193, 425, 177, 356, 883, 1002, 827, 662, 1929, 0, 0, 0, 0, 0, 0, 0, 920, 438, 1563, 733, 81, 1169, 141, 548, 0, 0, 0, 0, 0, 0, 0, 0, 515, 799, 182, 475, 3122, 293, 944, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 606, 1485, 782, 406, 210, 933, 997, 1032, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5990, 6077, 5533, 5695, 6454, 5593, 7359, 6451] avg=6144.0000 max=7359 max_rank=6 min=5533 min_rank=2 imbalance_ratio=1.197754 max_load_reduction=41793 max_load_reduction_pct=0.850281 moved_to_echo=25857 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=15 rerouted_tokens_per_expert_current_ep_rank=[413, 234, 235, 802, 297, 136, 204, 228, 2693, 0, 0, 0, 0, 0, 0, 0, 796, 1053, 800, 509, 617, 316, 386, 512, 224, 0, 0, 0, 0, 0, 0, 0, 1181, 737, 65, 1994, 943, 2232, 106, 1750, 0, 0, 0, 0, 0, 0, 0, 0, 1002, 20, 408, 1370, 1094, 197, 562, 106, 1458, 180, 188, 0, 0, 0, 0, 0, 27, 61, 911, 515, 1415, 894, 387, 500, 1136, 0, 0, 0, 0, 0, 0, 0, 1064, 613, 1156, 552, 224, 1191, 76, 241, 0, 0, 0, 0, 0, 0, 0, 0, 449, 1416, 166, 411, 1477, 177, 788, 726, 0, 0, 0, 0, 0, 0, 0, 0, 1406, 1007, 318, 93, 1175, 621, 1304, 607, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5242, 5213, 9008, 6585, 5846, 5117, 5610, 6531] avg=6144.0000 max=9008 max_rank=2 min=5117 min_rank=5 imbalance_ratio=1.466146 max_load_reduction=40144 max_load_reduction_pct=0.816732 moved_to_echo=23104 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=15 rerouted_tokens_per_expert_current_ep_rank=[244, 47, 159, 1113, 248, 643, 161, 710, 3276, 0, 0, 0, 0, 0, 0, 0, 358, 542, 1193, 767, 925, 1490, 1058, 362, 189, 0, 0, 0, 0, 0, 0, 0, 619, 465, 66, 781, 717, 1415, 839, 1267, 0, 0, 0, 0, 0, 0, 0, 0, 810, 150, 186, 1855, 850, 119, 1192, 53, 764, 251, 159, 0, 0, 0, 0, 0, 10, 33, 2154, 571, 1657, 82, 518, 311, 1584, 0, 0, 0, 0, 0, 0, 0, 1295, 1158, 1808, 621, 162, 474, 114, 92, 0, 0, 0, 0, 0, 0, 0, 0, 29, 859, 1551, 65, 3308, 246, 206, 469, 0, 0, 0, 0, 0, 0, 0, 0, 897, 853, 577, 502, 204, 447, 60, 192, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6601, 6884, 6169, 6389, 6920, 5724, 6733, 3732] avg=6144.0000 max=6920 max_rank=4 min=3732 min_rank=7 imbalance_ratio=1.126302 max_load_reduction=42232 max_load_reduction_pct=0.859212 moved_to_echo=23109 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=15 rerouted_tokens_per_expert_current_ep_rank=[118, 419, 147, 650, 435, 274, 251, 266, 2525, 0, 0, 0, 0, 0, 0, 0, 304, 518, 1523, 683, 351, 508, 712, 743, 568, 0, 0, 0, 0, 0, 0, 0, 425, 722, 447, 797, 555, 938, 110, 700, 0, 0, 0, 0, 0, 0, 0, 0, 226, 114, 554, 1383, 2312, 684, 157, 95, 524, 293, 477, 0, 0, 0, 0, 0, 198, 515, 1835, 626, 1110, 260, 804, 467, 1849, 0, 0, 0, 0, 0, 0, 0, 998, 1229, 983, 528, 232, 1022, 422, 111, 0, 0, 0, 0, 0, 0, 0, 0, 96, 495, 253, 558, 3082, 288, 1549, 363, 0, 0, 0, 0, 0, 0, 0, 0, 726, 2550, 813, 424, 609, 833, 594, 222, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5085, 5910, 4694, 6819, 7664, 5525, 6684, 6771] avg=6144.0000 max=7664 max_rank=4 min=4694 min_rank=2 imbalance_ratio=1.247396 max_load_reduction=41488 max_load_reduction_pct=0.844076 moved_to_echo=26644 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=15 rerouted_tokens_per_expert_current_ep_rank=[323, 804, 273, 676, 445, 452, 357, 368, 2904, 0, 0, 0, 0, 0, 0, 0, 242, 475, 2557, 420, 1090, 484, 1002, 371, 120, 0, 0, 0, 0, 0, 0, 0, 309, 635, 203, 781, 1064, 2126, 374, 948, 0, 0, 0, 0, 0, 0, 0, 0, 147, 258, 708, 1148, 869, 358, 532, 248, 380, 262, 101, 0, 0, 0, 0, 0, 293, 574, 550, 459, 1202, 175, 1641, 553, 1656, 0, 0, 0, 0, 0, 0, 0, 1147, 990, 1074, 502, 131, 384, 221, 907, 0, 0, 0, 0, 0, 0, 0, 0, 317, 892, 425, 176, 2885, 258, 730, 807, 0, 0, 0, 0, 0, 0, 0, 0, 1482, 542, 527, 230, 1406, 1013, 112, 77, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6602, 6761, 6440, 5011, 7103, 5356, 6490, 5389] avg=6144.0000 max=7103 max_rank=4 min=5011 min_rank=3 imbalance_ratio=1.156087 max_load_reduction=42049 max_load_reduction_pct=0.855489 moved_to_echo=24338 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4864, 4992, 2048, 8192, 7936, 9344, 3328, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9344, 9472, 12032, 3968, 2048, 7168, 2560, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3584, 5504, 12032, 4224, 5632, 5760, 5760, 4736, 2432, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=50048 observed_loss=-896 tokens_per_expert=[2304, 3328, 1664, 7296, 3456, 3072, 2432, 2944, 23424, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9984, 10752, 5120, 2432, 5888, 7040, 5632, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2048, 7808, 4224, 2816, 18688, 2304, 5888, 5888, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 896, 3200, 11264, 8832, 3072, 5504, 1792, 6016, 2304, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 2048, 7936, 3584, 7680, 3200, 5632, 4352, 14464, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=50048 slack_slots=256 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=16 tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 1402, 1399, 1725, 556, 347, 720, 423, 1662, 237, 567, 549, 1155, 378, 246, 125, 457, 367, 853, 74, 1865, 1321, 101, 386, 372, 263, 483, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 426, 178, 1046, 1395, 631, 1456, 923, 69, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 83, 286, 2292, 813, 1310, 1589, 467, 141] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=16 tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1573, 1956, 970, 445, 944, 420, 2944, 967, 119, 166, 1411, 219, 349, 276, 214, 938, 63, 519, 50, 908, 1195, 337, 231, 355, 3537, 75, 812, 66, 521, 574, 698, 73, 2249, 862, 1130, 205, 1401, 1415, 276, 196, 2255, 57, 199, 1077, 882, 442, 391, 21, 985, 1729, 86, 443, 2306, 316, 682, 506, 1167, 57] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=16 tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 1126, 1330, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 2199, 821, 796, 470, 557, 1322, 42, 1794, 85, 1630, 642, 144, 211, 876, 620, 69, 60, 119, 288, 609, 1245, 52, 426, 255, 661, 199, 1101, 1843, 282, 1826, 1857, 56, 241, 558, 391, 310, 681, 395, 109, 2295, 294, 761, 1158, 1179, 1145, 862, 862, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=16 tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 1188, 948, 1176, 1165, 295, 641, 182, 437, 159, 351, 1617, 892, 1429, 170, 417, 980, 346, 1056, 204, 944, 1220, 97, 129, 102, 862, 1049, 411, 115, 1028, 774, 906, 218, 891, 1357, 685, 638, 854, 2713, 154, 1064, 3509, 78, 311, 465, 1014, 544, 357, 60, 547, 1665, 64, 626, 926, 1046, 520, 348, 2326, 320] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=16 tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 2087, 1154, 2124, 263, 821, 780, 2125, 365, 267, 39, 1267, 1589, 260, 432, 278, 1194, 14, 402, 65, 1465, 665, 192, 403, 891, 1284, 13, 297, 58, 125, 403, 3076, 60, 832, 827, 448, 223, 1166, 2542, 1780, 2061, 2304, 99, 930, 338, 414, 507, 264, 27, 133, 2927, 390, 305, 826, 684, 790, 305, 542, 343] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=16 tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 3138, 1082, 1260, 570, 439, 460, 1106, 417, 83, 27, 1139, 900, 663, 502, 202, 1453, 59, 1677, 22, 3467, 725, 81, 492, 320, 800, 202, 373, 87, 682, 265, 475, 197, 737, 2281, 501, 234, 655, 1673, 372, 1586, 1758, 61, 954, 199, 1683, 566, 85, 139, 640, 2852, 62, 160, 817, 692, 1380, 304, 1648, 223] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=16 tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 2174, 1525, 1704, 307, 1656, 286, 781, 250, 33, 293, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 184, 758, 850, 114, 209, 245, 1367, 8, 367, 331, 824, 776, 1814, 217, 2118, 1913, 742, 469, 298, 2143, 742, 1079, 1346, 251, 87, 320, 1618, 607, 181, 328, 54, 1473, 189, 705, 1535, 349, 509, 278, 1959, 248] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=16 tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 1407, 790, 1451, 649, 93, 414, 1041, 583, 202, 52, 898, 1273, 1213, 94, 552, 748, 294, 1463, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1405, 6, 766, 638, 593, 399, 1926, 1101, 235, 455, 533, 2980, 83, 1361, 2802, 16, 311, 1410, 1869, 480, 480, 159, 173, 1649, 580, 700, 1146, 530, 268, 358, 930, 119] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=16 rerouted_tokens_per_expert_current_ep_rank=[605, 1765, 62, 801, 518, 141, 1071, 1956, 0, 0, 0, 0, 0, 0, 0, 0, 970, 445, 944, 420, 2944, 967, 119, 166, 282, 118, 0, 0, 0, 0, 0, 0, 1411, 219, 349, 276, 214, 938, 63, 519, 0, 0, 0, 0, 0, 0, 0, 0, 50, 908, 1195, 337, 231, 355, 3537, 75, 1113, 0, 0, 0, 0, 0, 0, 0, 812, 66, 521, 574, 698, 73, 2249, 862, 253, 0, 0, 0, 0, 0, 0, 0, 1130, 205, 1401, 1415, 276, 196, 315, 57, 0, 0, 0, 0, 0, 0, 0, 0, 199, 1077, 882, 442, 391, 21, 985, 1729, 131, 0, 0, 0, 0, 0, 0, 0, 86, 443, 2306, 316, 682, 506, 1167, 57, 545, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6919, 7375, 3989, 7801, 6108, 4995, 5857, 6108] avg=6144.0000 max=7801 max_rank=3 min=3989 min_rank=2 imbalance_ratio=1.269694 max_load_reduction=41351 max_load_reduction_pct=0.841288 moved_to_echo=23068 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=16 rerouted_tokens_per_expert_current_ep_rank=[279, 1423, 308, 689, 347, 220, 954, 1399, 0, 0, 0, 0, 0, 0, 0, 0, 1725, 556, 347, 720, 423, 1662, 237, 567, 115, 105, 0, 0, 0, 0, 0, 0, 549, 1155, 378, 246, 125, 457, 367, 853, 0, 0, 0, 0, 0, 0, 0, 0, 74, 1865, 1321, 101, 386, 372, 263, 483, 455, 0, 0, 0, 0, 0, 0, 0, 85, 190, 1376, 1074, 1017, 55, 1105, 1397, 226, 0, 0, 0, 0, 0, 0, 0, 426, 178, 1046, 1395, 631, 1456, 130, 69, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 1902, 769, 514, 1344, 55, 425, 2385, 117, 0, 0, 0, 0, 0, 0, 0, 83, 286, 2292, 813, 1310, 1589, 467, 141, 223, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5619, 6457, 4130, 5320, 6525, 5331, 8566, 7204] avg=6144.0000 max=8566 max_rank=6 min=4130 min_rank=2 imbalance_ratio=1.394206 max_load_reduction=40586 max_load_reduction_pct=0.825724 moved_to_echo=27626 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=16 rerouted_tokens_per_expert_current_ep_rank=[542, 2306, 258, 923, 476, 119, 767, 1330, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 637, 1502, 850, 1066, 856, 247, 1181, 232, 84, 0, 0, 0, 0, 0, 0, 2199, 821, 796, 470, 557, 1322, 42, 1794, 0, 0, 0, 0, 0, 0, 0, 0, 85, 1630, 642, 144, 211, 876, 620, 69, 916, 0, 0, 0, 0, 0, 0, 0, 60, 119, 288, 609, 1245, 52, 426, 255, 181, 0, 0, 0, 0, 0, 0, 0, 661, 199, 1101, 1843, 282, 1826, 260, 56, 0, 0, 0, 0, 0, 0, 0, 0, 241, 558, 391, 310, 681, 395, 109, 2295, 94, 0, 0, 0, 0, 0, 0, 0, 294, 761, 1158, 1179, 1145, 862, 862, 245, 449, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6721, 7745, 8001, 5193, 3235, 6228, 5074, 6955] avg=6144.0000 max=8001 max_rank=2 min=3235 min_rank=4 imbalance_ratio=1.302246 max_load_reduction=41151 max_load_reduction_pct=0.837219 moved_to_echo=21492 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=16 rerouted_tokens_per_expert_current_ep_rank=[1363, 1468, 93, 360, 830, 448, 793, 948, 0, 0, 0, 0, 0, 0, 0, 0, 1176, 1165, 295, 641, 182, 437, 159, 351, 444, 94, 0, 0, 0, 0, 0, 0, 1617, 892, 1429, 170, 417, 980, 346, 1056, 4, 0, 0, 0, 0, 0, 0, 0, 204, 944, 1220, 97, 129, 102, 862, 1049, 1737, 0, 0, 0, 0, 0, 0, 0, 411, 115, 1028, 774, 906, 218, 891, 1357, 196, 0, 0, 0, 0, 0, 0, 0, 685, 638, 854, 2713, 154, 1064, 477, 78, 0, 0, 0, 0, 0, 0, 0, 0, 311, 465, 1014, 544, 357, 60, 547, 1665, 101, 0, 0, 0, 0, 0, 0, 0, 64, 626, 926, 1046, 520, 348, 2326, 320, 851, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6303, 4944, 6911, 6344, 5896, 6663, 5064, 7027] avg=6144.0000 max=7027 max_rank=7 min=4944 min_rank=1 imbalance_ratio=1.143717 max_load_reduction=42125 max_load_reduction_pct=0.857035 moved_to_echo=24650 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=16 rerouted_tokens_per_expert_current_ep_rank=[471, 544, 859, 208, 560, 344, 1478, 1525, 0, 0, 0, 0, 0, 0, 0, 0, 1704, 307, 1656, 286, 781, 250, 33, 293, 168, 164, 0, 0, 0, 0, 0, 0, 1010, 1137, 926, 436, 556, 1214, 743, 1530, 1, 0, 0, 0, 0, 0, 0, 0, 184, 758, 850, 114, 209, 245, 1367, 8, 664, 0, 0, 0, 0, 0, 0, 0, 367, 331, 824, 776, 1814, 217, 2118, 1913, 350, 0, 0, 0, 0, 0, 0, 0, 742, 469, 298, 2143, 742, 1079, 189, 251, 0, 0, 0, 0, 0, 0, 0, 0, 87, 320, 1618, 607, 181, 328, 54, 1473, 181, 0, 0, 0, 0, 0, 0, 0, 189, 705, 1535, 349, 509, 278, 1959, 248, 325, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5989, 5642, 7553, 4399, 8710, 5913, 4849, 6097] avg=6144.0000 max=8710 max_rank=4 min=4399 min_rank=3 imbalance_ratio=1.417643 max_load_reduction=40442 max_load_reduction_pct=0.822795 moved_to_echo=25569 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=16 rerouted_tokens_per_expert_current_ep_rank=[364, 2338, 327, 28, 626, 171, 957, 790, 0, 0, 0, 0, 0, 0, 0, 0, 1451, 649, 93, 414, 1041, 583, 202, 52, 351, 106, 0, 0, 0, 0, 0, 0, 898, 1273, 1213, 94, 552, 748, 294, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 106, 1484, 713, 317, 395, 1365, 1010, 230, 1383, 0, 0, 0, 0, 0, 0, 0, 1405, 6, 766, 638, 593, 399, 1926, 1101, 227, 0, 0, 0, 0, 0, 0, 0, 235, 455, 533, 2980, 83, 1361, 391, 16, 0, 0, 0, 0, 0, 0, 0, 0, 311, 1410, 1869, 480, 480, 159, 173, 1649, 117, 0, 0, 0, 0, 0, 0, 0, 580, 700, 1146, 530, 268, 358, 930, 119, 677, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5601, 4942, 6535, 7003, 7061, 6054, 6648, 5308] avg=6144.0000 max=7061 max_rank=4 min=4942 min_rank=1 imbalance_ratio=1.149251 max_load_reduction=42091 max_load_reduction_pct=0.856344 moved_to_echo=25071 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=16 rerouted_tokens_per_expert_current_ep_rank=[378, 1021, 290, 336, 528, 134, 1419, 1154, 0, 0, 0, 0, 0, 0, 0, 0, 2124, 263, 821, 780, 2125, 365, 267, 39, 288, 157, 0, 0, 0, 0, 0, 0, 1267, 1589, 260, 432, 278, 1194, 14, 402, 1, 0, 0, 0, 0, 0, 0, 0, 65, 1465, 665, 192, 403, 891, 1284, 13, 1137, 0, 0, 0, 0, 0, 0, 0, 297, 58, 125, 403, 3076, 60, 832, 827, 336, 0, 0, 0, 0, 0, 0, 0, 448, 223, 1166, 2542, 1780, 2061, 322, 99, 0, 0, 0, 0, 0, 0, 0, 0, 930, 338, 414, 507, 264, 27, 133, 2927, 174, 0, 0, 0, 0, 0, 0, 0, 390, 305, 826, 684, 790, 305, 542, 343, 557, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5260, 7229, 5437, 6115, 6014, 8641, 5714, 4742] avg=6144.0000 max=8641 max_rank=5 min=4742 min_rank=7 imbalance_ratio=1.406413 max_load_reduction=40511 max_load_reduction_pct=0.824198 moved_to_echo=25111 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=16 rerouted_tokens_per_expert_current_ep_rank=[520, 1015, 418, 900, 462, 210, 2133, 1082, 0, 0, 0, 0, 0, 0, 0, 0, 1260, 570, 439, 460, 1106, 417, 83, 27, 220, 236, 0, 0, 0, 0, 0, 0, 1139, 900, 663, 502, 202, 1453, 59, 1677, 1, 0, 0, 0, 0, 0, 0, 0, 22, 3467, 725, 81, 492, 320, 800, 202, 868, 0, 0, 0, 0, 0, 0, 0, 373, 87, 682, 265, 475, 197, 737, 2281, 506, 0, 0, 0, 0, 0, 0, 0, 501, 234, 655, 1673, 372, 1586, 245, 61, 0, 0, 0, 0, 0, 0, 0, 0, 954, 199, 1683, 566, 85, 139, 640, 2852, 262, 0, 0, 0, 0, 0, 0, 0, 62, 160, 817, 692, 1380, 304, 1648, 223, 425, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6740, 4818, 6596, 6977, 5603, 5327, 7380, 5711] avg=6144.0000 max=7380 max_rank=6 min=4818 min_rank=1 imbalance_ratio=1.201172 max_load_reduction=41772 max_load_reduction_pct=0.849854 moved_to_echo=24021 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=2 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11520, 4608, 6144, 4608, 9728, 5632, 1408, 2688, 2176, 1152, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10112, 8064, 6016, 2688, 2944, 8320, 2048, 9344, 128, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[1792, 4096, 11008, 5632, 6656, 4608, 9984, 1792, 4096, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[4096, 6272, 8704, 4096, 3840, 1280, 3072, 17024, 1280, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[4608, 11904, 2688, 4352, 4352, 1792, 9600, 10240, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4864, 2688, 7168, 16768, 4352, 10752, 2432, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 1024, 5632, 5120, 9856, 1280, 10368, 10112, 2304, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 12544, 7424, 1408, 2560, 4608, 9856, 2176, 8320, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=17 tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 622, 1561, 531, 255, 366, 1724, 1518, 2373, 1648, 261, 417, 24, 375, 394, 400, 762, 1853, 224, 1392, 755, 601, 633, 2513, 253, 1660, 57, 1573, 1758, 523, 742, 176, 89, 1010, 905, 260, 166, 3850, 458, 807, 764, 269, 925, 831, 551, 555, 946, 654, 445, 447, 524, 310, 516, 1965, 460, 1208, 275] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=17 tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 365, 2823, 180, 712, 346, 1995, 53, 1112, 801, 184, 1201, 259, 393, 165, 931, 1439, 471, 535, 162, 1153, 184, 133, 2267, 709, 765, 241, 1018, 1348, 117, 624, 686, 41, 748, 2202, 392, 263, 2206, 2327, 1074, 317, 598, 858, 188, 2083, 591, 669, 321, 790, 522, 430, 788, 207, 1170, 526, 2279, 198] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=17 tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 654, 1280, 785, 1851, 329, 2904, 814, 1387, 2601, 1141, 422, 68, 370, 518, 479, 272, 1517, 261, 736, 502, 812, 512, 1777, 120, 671, 111, 153, 497, 228, 628, 1089, 127, 117, 1030, 159, 1537, 2368, 983, 1434, 313, 822, 575, 55, 2560, 125, 155, 339, 951, 92, 422, 433, 1706, 432, 708, 1222, 442] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=17 tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 891, 564, 294, 807, 179, 1726, 589, 1243, 1674, 1377, 439, 408, 852, 126, 398, 67, 1515, 358, 555, 361, 92, 606, 1677, 710, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 227, 1637, 211, 256, 2438, 1045, 342, 1049, 639, 207, 146, 850, 1523, 190, 68, 616, 110, 102, 201, 543, 2125, 312, 2263, 1195] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=17 tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1463, 168, 648, 1609, 279, 270, 57, 1307, 2206, 669, 342, 117, 735, 349, 1372, 1789, 906, 588, 408, 196, 51, 333, 2055, 161, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 253, 2617, 83, 160, 2604, 1792, 1485, 391, 202, 95, 154, 2901, 246, 96, 226, 1685, 88, 821, 31, 1120, 1347, 265, 833, 149] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=17 tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 756, 1058, 612, 701, 222, 1967, 289, 738, 2151, 567, 190, 535, 167, 1022, 869, 898, 1594, 235, 356, 812, 78, 651, 1810, 199, 200, 198, 2302, 285, 535, 232, 434, 213, 314, 2147, 1170, 363, 1239, 2377, 2547, 368, 69, 454, 145, 537, 39, 586, 452, 454, 257, 851, 164, 1264, 1933, 1186, 610, 217] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=17 tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 423, 2633, 520, 416, 1068, 1700, 570, 1003, 1583, 273, 727, 470, 613, 427, 793, 2115, 1782, 935, 1403, 370, 272, 686, 1330, 137, 522, 289, 1072, 801, 175, 818, 682, 105, 418, 1185, 429, 332, 2282, 1898, 919, 1304, 134, 641, 425, 1256, 149, 184, 536, 475, 1374, 248, 397, 440, 1364, 1134, 1072, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=17 tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 487, 1580, 909, 2133, 146, 1322, 92, 613, 1456, 1151, 420, 217, 250, 168, 1237, 418, 752, 1146, 1305, 528, 178, 140, 1729, 91, 743, 525, 703, 1766, 650, 815, 803, 31, 233, 1561, 368, 977, 1582, 2156, 1526, 911, 625, 128, 186, 2012, 288, 210, 135, 708, 132, 335, 397, 1187, 1880, 656, 578, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=17 rerouted_tokens_per_expert_current_ep_rank=[18, 221, 118, 501, 320, 87, 410, 343, 2775, 0, 0, 0, 0, 0, 0, 0, 622, 1561, 531, 255, 366, 276, 1518, 2373, 0, 0, 0, 0, 0, 0, 0, 0, 1648, 261, 417, 24, 375, 394, 400, 762, 113, 41, 0, 0, 0, 0, 0, 0, 1853, 224, 1392, 755, 601, 633, 2511, 253, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 57, 1573, 1758, 523, 742, 176, 89, 592, 0, 0, 0, 0, 0, 0, 0, 1010, 690, 260, 166, 0, 458, 807, 764, 0, 0, 0, 0, 0, 0, 0, 0, 269, 925, 831, 551, 555, 946, 654, 445, 815, 1075, 0, 0, 0, 0, 0, 0, 447, 524, 310, 516, 1965, 460, 1208, 275, 102, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4793, 7502, 4435, 8222, 7170, 4155, 7066, 5809] avg=6144.0000 max=8222 max_rank=3 min=4155 min_rank=5 imbalance_ratio=1.338216 max_load_reduction=40930 max_load_reduction_pct=0.832723 moved_to_echo=24200 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 66, 120, 448, 1456, 617, 756, 1074, 1706, 0, 0, 0, 0, 0, 0, 0, 654, 1280, 785, 1851, 329, 465, 814, 1387, 0, 0, 0, 0, 0, 0, 0, 0, 2601, 1141, 422, 68, 370, 518, 479, 272, 128, 69, 0, 0, 0, 0, 0, 0, 1517, 261, 736, 502, 812, 512, 1776, 120, 0, 0, 0, 0, 0, 0, 0, 0, 671, 111, 153, 497, 228, 628, 1089, 127, 997, 0, 0, 0, 0, 0, 0, 0, 117, 786, 159, 1537, 0, 983, 1434, 313, 0, 0, 0, 0, 0, 0, 0, 0, 822, 575, 55, 2560, 125, 155, 339, 951, 1373, 662, 0, 0, 0, 0, 0, 0, 92, 422, 433, 1706, 432, 708, 1222, 442, 116, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6262, 7565, 6068, 6236, 4501, 5329, 7617, 5574] avg=6144.0000 max=7617 max_rank=6 min=4501 min_rank=4 imbalance_ratio=1.239746 max_load_reduction=41535 max_load_reduction_pct=0.845032 moved_to_echo=23021 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=17 rerouted_tokens_per_expert_current_ep_rank=[4, 214, 170, 291, 1867, 523, 545, 378, 1589, 0, 0, 0, 0, 0, 0, 0, 365, 2823, 180, 712, 346, 319, 53, 1112, 0, 0, 0, 0, 0, 0, 0, 0, 801, 184, 1201, 259, 393, 165, 931, 1439, 275, 48, 0, 0, 0, 0, 0, 0, 471, 535, 162, 1153, 184, 133, 2265, 709, 0, 0, 0, 0, 0, 0, 0, 0, 765, 241, 1018, 1348, 117, 624, 686, 41, 685, 0, 0, 0, 0, 0, 0, 0, 748, 1678, 392, 263, 0, 2327, 1074, 317, 0, 0, 0, 0, 0, 0, 0, 0, 598, 858, 188, 2083, 591, 669, 321, 790, 943, 617, 0, 0, 0, 0, 0, 0, 522, 430, 788, 207, 1170, 526, 2279, 198, 249, 2, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5581, 5910, 5696, 5612, 5525, 6799, 7658, 6371] avg=6144.0000 max=7658 max_rank=6 min=5525 min_rank=4 imbalance_ratio=1.246419 max_load_reduction=41494 max_load_reduction_pct=0.844198 moved_to_echo=26353 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=17 rerouted_tokens_per_expert_current_ep_rank=[28, 167, 125, 319, 1390, 417, 738, 903, 1877, 0, 0, 0, 0, 0, 0, 0, 1463, 168, 648, 1609, 279, 45, 57, 1307, 0, 0, 0, 0, 0, 0, 0, 0, 2206, 669, 342, 117, 735, 349, 1372, 1789, 327, 6, 0, 0, 0, 0, 0, 0, 906, 588, 408, 196, 51, 333, 2054, 161, 0, 0, 0, 0, 0, 0, 0, 0, 1262, 341, 1162, 1376, 118, 2351, 723, 10, 92, 0, 0, 0, 0, 0, 0, 0, 253, 1994, 83, 160, 0, 1792, 1485, 391, 0, 0, 0, 0, 0, 0, 0, 0, 202, 95, 154, 2901, 246, 96, 226, 1685, 127, 727, 0, 0, 0, 0, 0, 0, 88, 821, 31, 1120, 1347, 265, 833, 149, 296, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5964, 5576, 7912, 4697, 7435, 6158, 6459, 4951] avg=6144.0000 max=7912 max_rank=2 min=4697 min_rank=3 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=25003 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=17 rerouted_tokens_per_expert_current_ep_rank=[19, 246, 43, 103, 3878, 1018, 338, 391, 1756, 0, 0, 0, 0, 0, 0, 0, 891, 564, 294, 807, 179, 277, 589, 1243, 0, 0, 0, 0, 0, 0, 0, 0, 1674, 1377, 439, 408, 852, 126, 398, 67, 204, 41, 0, 0, 0, 0, 0, 0, 1515, 358, 555, 361, 92, 606, 1676, 710, 0, 0, 0, 0, 0, 0, 0, 0, 870, 696, 1081, 1197, 440, 1254, 1765, 10, 592, 0, 0, 0, 0, 0, 0, 0, 227, 1248, 211, 256, 0, 1045, 342, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 639, 207, 146, 850, 1523, 190, 68, 616, 816, 682, 0, 0, 0, 0, 0, 0, 110, 102, 201, 543, 2125, 312, 2263, 1195, 185, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7792, 4844, 5586, 5873, 7905, 4378, 5737, 7037] avg=6144.0000 max=7905 max_rank=4 min=4378 min_rank=5 imbalance_ratio=1.286621 max_load_reduction=41247 max_load_reduction_pct=0.839172 moved_to_echo=25057 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=17 rerouted_tokens_per_expert_current_ep_rank=[196, 707, 534, 505, 899, 1721, 655, 1316, 893, 0, 0, 0, 0, 0, 0, 0, 756, 1058, 612, 701, 222, 315, 289, 738, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 567, 190, 535, 167, 1022, 869, 898, 268, 47, 0, 0, 0, 0, 0, 0, 1594, 235, 356, 812, 78, 651, 1809, 199, 0, 0, 0, 0, 0, 0, 0, 0, 200, 198, 2302, 285, 535, 232, 434, 213, 675, 0, 0, 0, 0, 0, 0, 0, 314, 1637, 1170, 363, 0, 2377, 2547, 368, 0, 0, 0, 0, 0, 0, 0, 0, 69, 454, 145, 537, 39, 586, 452, 454, 930, 346, 0, 0, 0, 0, 0, 0, 257, 851, 164, 1264, 1933, 1186, 610, 217, 242, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7426, 4691, 6714, 5734, 5074, 8776, 4012, 6725] avg=6144.0000 max=8776 max_rank=5 min=4012 min_rank=6 imbalance_ratio=1.428385 max_load_reduction=40376 max_load_reduction_pct=0.821452 moved_to_echo=24587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=17 rerouted_tokens_per_expert_current_ep_rank=[45, 136, 251, 290, 872, 909, 46, 783, 1645, 0, 0, 0, 0, 0, 0, 0, 423, 2633, 520, 416, 1068, 259, 570, 1003, 0, 0, 0, 0, 0, 0, 0, 0, 1583, 273, 727, 470, 613, 427, 793, 2115, 152, 45, 0, 0, 0, 0, 0, 0, 1782, 935, 1403, 370, 272, 686, 1325, 137, 0, 0, 0, 0, 0, 0, 0, 0, 522, 289, 1072, 801, 175, 818, 682, 105, 588, 0, 0, 0, 0, 0, 0, 0, 418, 896, 429, 332, 0, 1898, 919, 1304, 0, 0, 0, 0, 0, 0, 0, 0, 134, 641, 425, 1256, 149, 184, 536, 475, 808, 637, 0, 0, 0, 0, 0, 0, 1374, 248, 397, 440, 1364, 1134, 1072, 511, 137, 5, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4977, 6892, 7198, 6910, 5052, 6196, 5245, 6682] avg=6144.0000 max=7198 max_rank=2 min=4977 min_rank=0 imbalance_ratio=1.171549 max_load_reduction=41954 max_load_reduction_pct=0.853556 moved_to_echo=23175 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=17 rerouted_tokens_per_expert_current_ep_rank=[3, 290, 88, 249, 1523, 1045, 1404, 614, 1141, 0, 0, 0, 0, 0, 0, 0, 487, 1580, 909, 2133, 146, 212, 92, 613, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 1151, 420, 217, 250, 168, 1237, 418, 195, 31, 0, 0, 0, 0, 0, 0, 752, 1146, 1305, 528, 178, 140, 1728, 91, 0, 0, 0, 0, 0, 0, 0, 0, 743, 525, 703, 1766, 650, 815, 803, 31, 454, 0, 0, 0, 0, 0, 0, 0, 233, 1190, 368, 977, 0, 2156, 1526, 911, 0, 0, 0, 0, 0, 0, 0, 0, 625, 128, 186, 2012, 288, 210, 135, 708, 625, 441, 0, 0, 0, 0, 0, 0, 132, 335, 397, 1187, 1880, 656, 578, 661, 176, 1, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6357, 6172, 5543, 5868, 6490, 7361, 5358, 6003] avg=6144.0000 max=7361 max_rank=5 min=5358 min_rank=6 imbalance_ratio=1.198079 max_load_reduction=41791 max_load_reduction_pct=0.850240 moved_to_echo=25212 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3328, 10240, 3072, 4096, 0, 13056, 10240, 5504, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5760, 11776, 4480, 8576, 2944, 2176, 4096, 9856, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[14208, 5632, 4224, 2176, 3840, 3200, 6528, 7808, 1664, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3456, 3968, 2176, 12800, 3584, 3072, 2816, 6144, 6528, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[384, 2048, 1536, 2816, 12288, 6400, 4992, 5888, 13440, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3072, 3840, 2816, 7040, 12288, 5248, 10112, 3712, 1536, 128, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2560, 9088, 9088, 2816, 7552, 6400, 640, 4736, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[10496, 4352, 6400, 4736, 2304, 3712, 15232, 2432, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=18 tokens_per_expert_current_ep_rank=[1190, 527, 3546, 1032, 292, 546, 2702, 1249, 186, 2121, 751, 194, 262, 516, 273, 184, 33, 518, 775, 48, 593, 1192, 241, 776, 450, 79, 967, 1216, 883, 286, 296, 279, 679, 1281, 63, 315, 851, 995, 212, 4183, 549, 766, 1685, 164, 904, 215, 578, 200, 30, 813, 176, 1073, 807, 474, 1679, 547, 32, 192, 987, 1609, 983, 757, 1007, 143] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=18 tokens_per_expert_current_ep_rank=[774, 399, 1942, 2779, 763, 150, 804, 2587, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 35, 391, 1714, 260, 678, 902, 318, 245, 627, 38, 220, 1539, 343, 751, 390, 523, 2281, 264, 160, 156, 577, 867, 9, 2060, 542, 625, 284, 66, 1036, 218, 755, 547, 356, 1070, 87, 537, 666, 13, 2014, 1016, 101, 358, 104, 1966, 62, 94, 1650, 2102] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=18 tokens_per_expert_current_ep_rank=[463, 470, 2682, 911, 224, 1392, 2190, 861, 89, 1843, 1114, 115, 318, 841, 856, 1321, 331, 308, 359, 267, 1330, 2640, 1607, 240, 155, 53, 55, 1503, 323, 390, 108, 512, 1814, 1739, 16, 932, 166, 988, 97, 1697, 191, 404, 669, 9, 1203, 668, 811, 649, 273, 858, 95, 1435, 997, 18, 1427, 1480, 58, 93, 292, 3218, 83, 310, 84, 507] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=18 tokens_per_expert_current_ep_rank=[305, 2006, 1080, 838, 902, 635, 1370, 2339, 612, 2109, 788, 54, 333, 341, 710, 1481, 1021, 271, 1411, 146, 478, 685, 577, 238, 449, 169, 234, 962, 370, 1162, 313, 179, 978, 2311, 796, 443, 373, 433, 690, 1198, 1105, 272, 194, 73, 749, 32, 1583, 604, 1992, 774, 510, 188, 580, 9, 810, 752, 472, 1106, 230, 1897, 894, 2117, 122, 297] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=18 tokens_per_expert_current_ep_rank=[619, 631, 3329, 816, 273, 1020, 922, 1363, 739, 2213, 387, 71, 687, 192, 689, 778, 226, 274, 543, 144, 2239, 1991, 2019, 300, 329, 49, 356, 2712, 673, 642, 585, 1160, 1662, 151, 3, 5, 231, 1004, 49, 3171, 179, 258, 442, 35, 1744, 103, 678, 239, 283, 442, 164, 221, 338, 5, 1891, 730, 29, 441, 785, 3316, 224, 429, 418, 511] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=18 tokens_per_expert_current_ep_rank=[788, 2534, 4631, 1148, 847, 338, 763, 447, 365, 1263, 680, 496, 423, 279, 119, 538, 21, 646, 674, 972, 1402, 1107, 933, 405, 249, 39, 69, 3161, 325, 233, 538, 880, 1123, 2541, 50, 149, 52, 1084, 57, 1625, 486, 439, 642, 12, 951, 149, 1649, 189, 205, 1213, 42, 1599, 89, 32, 1744, 606, 125, 403, 146, 2652, 613, 934, 171, 67] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=18 tokens_per_expert_current_ep_rank=[1292, 1398, 4306, 1064, 15, 1545, 1516, 1097, 44, 2354, 869, 7, 241, 776, 237, 516, 73, 1246, 879, 434, 747, 639, 2149, 980, 157, 5, 726, 1871, 1226, 189, 137, 744, 1013, 443, 44, 31, 24, 344, 16, 2852, 35, 96, 715, 306, 663, 56, 2269, 102, 404, 1732, 30, 1606, 140, 34, 2163, 654, 49, 42, 810, 1635, 137, 593, 208, 427] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=18 tokens_per_expert_current_ep_rank=[897, 836, 1547, 1178, 1184, 280, 2509, 963, 206, 2031, 1307, 45, 276, 228, 1084, 808, 216, 485, 2224, 15, 2224, 1122, 1387, 145, 322, 293, 177, 642, 306, 437, 120, 332, 911, 200, 387, 604, 516, 557, 103, 887, 90, 176, 353, 13, 1382, 46, 1413, 672, 564, 1184, 233, 497, 500, 23, 2158, 561, 1075, 1727, 110, 3601, 896, 621, 1127, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=18 rerouted_tokens_per_expert_current_ep_rank=[774, 399, 0, 2779, 763, 150, 187, 2587, 0, 0, 0, 0, 0, 0, 0, 0, 109, 1850, 183, 164, 1799, 190, 1568, 1474, 225, 95, 0, 0, 0, 0, 0, 0, 35, 391, 1714, 260, 678, 873, 318, 245, 0, 0, 0, 0, 0, 0, 0, 0, 627, 38, 220, 1539, 343, 751, 390, 523, 745, 302, 0, 0, 0, 0, 0, 0, 2281, 264, 160, 156, 577, 867, 9, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 542, 625, 284, 66, 1036, 218, 755, 547, 1197, 0, 0, 0, 0, 0, 0, 0, 356, 1070, 87, 537, 666, 13, 2014, 1016, 220, 0, 0, 0, 0, 0, 0, 0, 101, 358, 104, 1966, 62, 94, 1650, 2102, 49, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7639, 7657, 4514, 5478, 6100, 5270, 5979, 6515] avg=6144.0000 max=7657 max_rank=1 min=4514 min_rank=2 imbalance_ratio=1.246257 max_load_reduction=41495 max_load_reduction_pct=0.844218 moved_to_echo=23864 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=18 rerouted_tokens_per_expert_current_ep_rank=[463, 470, 0, 911, 224, 1392, 507, 861, 0, 0, 0, 0, 0, 0, 0, 0, 89, 1843, 1114, 115, 318, 841, 856, 1321, 185, 260, 0, 0, 0, 0, 0, 0, 331, 308, 359, 267, 1330, 2554, 1607, 240, 0, 0, 0, 0, 0, 0, 0, 0, 155, 53, 55, 1503, 323, 390, 108, 512, 1028, 822, 0, 0, 0, 0, 0, 0, 1814, 1739, 16, 932, 166, 988, 97, 1471, 0, 0, 0, 0, 0, 0, 0, 0, 191, 404, 669, 9, 1203, 668, 811, 649, 1654, 0, 0, 0, 0, 0, 0, 0, 273, 858, 95, 1435, 997, 18, 1427, 1480, 601, 0, 0, 0, 0, 0, 0, 0, 58, 93, 292, 3218, 83, 310, 84, 507, 41, 86, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4828, 6942, 6996, 4949, 7223, 6258, 7184, 4772] avg=6144.0000 max=7223 max_rank=4 min=4772 min_rank=7 imbalance_ratio=1.175618 max_load_reduction=41929 max_load_reduction_pct=0.853048 moved_to_echo=25437 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1190, 527, 0, 1032, 292, 546, 624, 1249, 0, 0, 0, 0, 0, 0, 0, 0, 186, 2121, 751, 194, 262, 516, 273, 184, 457, 321, 0, 0, 0, 0, 0, 0, 33, 518, 775, 48, 593, 1153, 241, 776, 0, 0, 0, 0, 0, 0, 0, 0, 450, 79, 967, 1216, 883, 286, 296, 279, 1359, 1015, 0, 0, 0, 0, 0, 0, 679, 1281, 63, 315, 851, 995, 212, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 549, 766, 1685, 164, 904, 215, 578, 200, 2187, 0, 0, 0, 0, 0, 0, 0, 30, 813, 176, 1073, 807, 474, 1679, 547, 742, 0, 0, 0, 0, 0, 0, 0, 32, 192, 987, 1609, 983, 757, 1007, 143, 101, 39, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5460, 5265, 4137, 6830, 8021, 7248, 6341, 5850] avg=6144.0000 max=8021 max_rank=4 min=4137 min_rank=2 imbalance_ratio=1.305501 max_load_reduction=41131 max_load_reduction_pct=0.836812 moved_to_echo=27460 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=18 rerouted_tokens_per_expert_current_ep_rank=[305, 2006, 0, 838, 902, 635, 304, 2339, 0, 0, 0, 0, 0, 0, 0, 0, 612, 2109, 788, 54, 333, 341, 710, 1481, 134, 168, 0, 0, 0, 0, 0, 0, 1021, 271, 1411, 146, 478, 659, 577, 238, 0, 0, 0, 0, 0, 0, 0, 0, 449, 169, 234, 962, 370, 1162, 313, 179, 415, 519, 0, 0, 0, 0, 0, 0, 978, 2311, 796, 443, 373, 433, 690, 1032, 0, 0, 0, 0, 0, 0, 0, 0, 1105, 272, 194, 73, 749, 32, 1583, 604, 665, 0, 0, 0, 0, 0, 0, 0, 1992, 774, 510, 188, 580, 9, 810, 752, 379, 0, 0, 0, 0, 0, 0, 0, 472, 1106, 230, 1897, 894, 2117, 122, 297, 32, 26, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7329, 6730, 4801, 4772, 7056, 5277, 5994, 7193] avg=6144.0000 max=7329 max_rank=0 min=4772 min_rank=3 imbalance_ratio=1.192871 max_load_reduction=41823 max_load_reduction_pct=0.850891 moved_to_echo=25520 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=18 rerouted_tokens_per_expert_current_ep_rank=[1292, 1398, 0, 1064, 15, 1545, 351, 1097, 0, 0, 0, 0, 0, 0, 0, 0, 44, 2354, 869, 7, 241, 776, 237, 516, 312, 180, 0, 0, 0, 0, 0, 0, 73, 1246, 879, 434, 747, 619, 2149, 980, 0, 0, 0, 0, 0, 0, 0, 0, 157, 5, 726, 1871, 1226, 189, 137, 744, 1651, 569, 0, 0, 0, 0, 0, 0, 1013, 443, 44, 31, 24, 344, 16, 2472, 0, 0, 0, 0, 0, 0, 0, 0, 35, 96, 715, 306, 663, 56, 2269, 102, 2655, 0, 0, 0, 0, 0, 0, 0, 404, 1732, 30, 1606, 140, 34, 2163, 654, 416, 0, 0, 0, 0, 0, 0, 0, 49, 42, 810, 1635, 137, 593, 208, 427, 68, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6762, 5536, 7127, 7275, 4387, 6897, 7179, 3989] avg=6144.0000 max=7275 max_rank=3 min=3989 min_rank=7 imbalance_ratio=1.184082 max_load_reduction=41877 max_load_reduction_pct=0.851990 moved_to_echo=22452 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=18 rerouted_tokens_per_expert_current_ep_rank=[897, 836, 0, 1178, 1184, 280, 580, 963, 0, 0, 0, 0, 0, 0, 0, 0, 206, 2031, 1307, 45, 276, 228, 1084, 808, 97, 298, 0, 0, 0, 0, 0, 0, 216, 485, 2224, 15, 2224, 1086, 1387, 145, 0, 0, 0, 0, 0, 0, 0, 0, 322, 293, 177, 642, 306, 437, 120, 332, 593, 942, 0, 0, 0, 0, 0, 0, 911, 200, 387, 604, 516, 557, 103, 769, 0, 0, 0, 0, 0, 0, 0, 0, 90, 176, 353, 13, 1382, 46, 1413, 672, 954, 0, 0, 0, 0, 0, 0, 0, 564, 1184, 233, 497, 500, 23, 2158, 561, 689, 0, 0, 0, 0, 0, 0, 0, 1075, 1727, 110, 3601, 896, 621, 1127, 139, 21, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5918, 6380, 7782, 4164, 4047, 5099, 6409, 9353] avg=6144.0000 max=9353 max_rank=7 min=4047 min_rank=4 imbalance_ratio=1.522298 max_load_reduction=39799 max_load_reduction_pct=0.809713 moved_to_echo=24908 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=18 rerouted_tokens_per_expert_current_ep_rank=[788, 2534, 0, 1148, 847, 338, 178, 447, 0, 0, 0, 0, 0, 0, 0, 0, 365, 1263, 680, 496, 423, 279, 119, 538, 177, 90, 0, 0, 0, 0, 0, 0, 21, 646, 674, 972, 1402, 1071, 933, 405, 0, 0, 0, 0, 0, 0, 0, 0, 249, 39, 69, 3161, 325, 233, 538, 880, 1776, 286, 0, 0, 0, 0, 0, 0, 1123, 2541, 50, 149, 52, 1084, 57, 1409, 0, 0, 0, 0, 0, 0, 0, 0, 486, 439, 642, 12, 951, 149, 1649, 189, 2855, 0, 0, 0, 0, 0, 0, 0, 205, 1213, 42, 1599, 89, 32, 1744, 606, 209, 0, 0, 0, 0, 0, 0, 0, 125, 403, 146, 2652, 613, 934, 171, 67, 39, 36, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6280, 4430, 6124, 7556, 6465, 7372, 5739, 5186] avg=6144.0000 max=7556 max_rank=3 min=4430 min_rank=1 imbalance_ratio=1.229818 max_load_reduction=41596 max_load_reduction_pct=0.846273 moved_to_echo=24762 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=18 rerouted_tokens_per_expert_current_ep_rank=[619, 631, 0, 816, 273, 1020, 214, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 739, 2213, 387, 71, 687, 192, 689, 778, 347, 109, 0, 0, 0, 0, 0, 0, 226, 274, 543, 144, 2239, 1926, 2019, 300, 0, 0, 0, 0, 0, 0, 0, 0, 329, 49, 356, 2712, 673, 642, 585, 1160, 1276, 346, 0, 0, 0, 0, 0, 0, 1662, 151, 3, 5, 231, 1004, 49, 2748, 0, 0, 0, 0, 0, 0, 0, 0, 179, 258, 442, 35, 1744, 103, 678, 239, 2053, 0, 0, 0, 0, 0, 0, 0, 283, 442, 164, 221, 338, 5, 1891, 730, 253, 0, 0, 0, 0, 0, 0, 0, 29, 441, 785, 3316, 224, 429, 418, 511, 76, 65, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4936, 6212, 7671, 8128, 5853, 5731, 4327, 6294] avg=6144.0000 max=8128 max_rank=3 min=4327 min_rank=6 imbalance_ratio=1.322917 max_load_reduction=41024 max_load_reduction_pct=0.834635 moved_to_echo=22205 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2048, 4224, 8704, 2304, 9728, 9984, 9344, 3456, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[4224, 8192, 1408, 7168, 4224, 640, 13952, 6400, 3584, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[6400, 8832, 0, 9856, 4608, 6016, 3072, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 3072, 4992, 768, 8704, 1536, 9856, 3328, 14336, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 15872, 6144, 1152, 4352, 3456, 5632, 7168, 2048, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2048, 4480, 3584, 19968, 3968, 5888, 4864, 4224, 512, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 768, 2816, 13696, 4480, 4096, 2560, 4736, 8960, 4864, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[10496, 8960, 1536, 2688, 2816, 6272, 1280, 15360, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49408 slack_slots=896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=19 tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 742, 1021, 662, 600, 2061, 2759, 2504, 3718, 30, 1615, 373, 135, 1188, 90, 325, 1475, 1120, 595, 1238, 488, 444, 735, 320, 170, 464, 386, 1354, 929, 482, 444, 1835, 1860, 360, 496, 477, 28, 2183, 492, 159, 112, 266, 164, 482, 293, 94, 221, 239, 140, 875, 463, 317, 700, 117, 356, 419, 1291] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=19 tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 562, 415, 2707, 592, 1890, 2541, 1566, 1502, 261, 3199, 47, 1078, 2551, 164, 411, 1466, 655, 33, 1230, 352, 582, 706, 583, 121, 629, 195, 808, 336, 60, 605, 775, 815, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 1319, 114, 173, 165, 30, 124, 98, 613, 1335, 1168, 148, 232, 81, 71, 500, 659] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=19 tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 576, 903, 2075, 1257, 437, 2702, 3189, 329, 396, 1280, 143, 96, 1785, 227, 707, 2257, 101, 212, 1032, 204, 41, 276, 19, 466, 1632, 267, 779, 1009, 10, 385, 650, 373, 222, 291, 462, 183, 880, 665, 571, 1140, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 1095, 609, 487, 437, 38, 59, 626, 185] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=19 tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 179, 2839, 1939, 1631, 1304, 2403, 989, 1019, 160, 3960, 71, 607, 1073, 175, 321, 472, 719, 1154, 529, 157, 227, 355, 220, 313, 977, 551, 821, 784, 181, 705, 3642, 1296, 378, 37, 331, 265, 934, 976, 595, 1318, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 342, 678, 564, 130, 12, 24, 208, 10] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=19 tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 366, 1443, 430, 1068, 1154, 2587, 1551, 1553, 53, 974, 157, 524, 943, 221, 610, 289, 454, 650, 3069, 200, 846, 68, 2263, 272, 1279, 9, 343, 1365, 138, 1143, 499, 279, 38, 463, 401, 197, 709, 320, 1064, 270, 1702, 494, 1141, 494, 149, 106, 73, 1724, 1253, 1431, 1306, 196, 13, 360, 317, 302] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=19 tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 88, 3902, 314, 289, 1234, 391, 1513, 2538, 202, 2930, 415, 490, 2300, 113, 620, 2167, 1222, 983, 1930, 317, 246, 147, 67, 254, 829, 423, 1302, 1373, 301, 1063, 463, 1597, 479, 42, 106, 716, 1126, 1145, 94, 1159, 835, 9, 2604, 768, 417, 955, 15, 748, 368, 836, 207, 67, 49, 97, 266, 520] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=19 tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 418, 1590, 1019, 174, 860, 1303, 1298, 175, 80, 2815, 72, 105, 1923, 854, 740, 2223, 2419, 525, 608, 229, 951, 288, 165, 273, 759, 346, 457, 1212, 847, 230, 2343, 401, 164, 611, 192, 272, 2644, 894, 687, 2224, 230, 54, 635, 313, 123, 158, 8, 773, 1435, 753, 195, 298, 12, 46, 47, 1003] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=19 tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 534, 1369, 875, 252, 508, 1735, 1558, 715, 110, 907, 259, 647, 1625, 902, 964, 1559, 689, 219, 1924, 393, 481, 57, 137, 615, 192, 480, 482, 1226, 710, 1263, 3434, 1128, 480, 284, 536, 678, 2396, 418, 41, 1549, 916, 27, 780, 800, 473, 115, 147, 1196, 2838, 588, 1151, 111, 87, 291, 304, 278] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 873, 118, 723, 435, 172, 2110, 1749, 292, 0, 0, 0, 0, 0, 0, 0, 576, 591, 2075, 1257, 437, 0, 0, 329, 0, 0, 0, 0, 0, 0, 0, 0, 396, 717, 143, 96, 1785, 227, 707, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 101, 212, 1032, 204, 41, 276, 19, 466, 2431, 0, 0, 0, 0, 0, 0, 0, 1632, 267, 779, 1009, 10, 385, 422, 373, 0, 0, 0, 0, 0, 0, 0, 0, 222, 291, 462, 183, 880, 665, 571, 1140, 228, 20, 0, 0, 0, 0, 0, 0, 1500, 120, 1614, 2047, 979, 723, 412, 1384, 563, 428, 0, 0, 0, 0, 0, 0, 1095, 609, 487, 437, 38, 59, 626, 185, 2702, 330, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6900, 5265, 6328, 4782, 4877, 4662, 9770, 6568] avg=6144.0000 max=9770 max_rank=6 min=4662 min_rank=5 imbalance_ratio=1.590169 max_load_reduction=39382 max_load_reduction_pct=0.801229 moved_to_echo=25877 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=19 rerouted_tokens_per_expert_current_ep_rank=[1104, 3380, 124, 351, 239, 140, 194, 714, 331, 0, 0, 0, 0, 0, 0, 0, 742, 667, 662, 600, 2061, 0, 0, 3718, 0, 0, 0, 0, 0, 0, 0, 0, 30, 905, 373, 135, 1188, 90, 325, 1475, 0, 0, 0, 0, 0, 0, 0, 0, 1120, 595, 1238, 488, 444, 735, 320, 170, 1907, 0, 0, 0, 0, 0, 0, 0, 464, 386, 1354, 929, 482, 444, 1189, 1860, 0, 0, 0, 0, 0, 0, 0, 0, 360, 496, 477, 28, 2183, 492, 159, 112, 646, 23, 0, 0, 0, 0, 0, 0, 266, 164, 482, 293, 94, 221, 239, 140, 710, 336, 0, 0, 0, 0, 0, 0, 875, 463, 317, 700, 117, 356, 419, 1291, 2759, 261, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6577, 8450, 4521, 7017, 7108, 4976, 2945, 7558] avg=6144.0000 max=8450 max_rank=1 min=2945 min_rank=6 imbalance_ratio=1.375326 max_load_reduction=40702 max_load_reduction_pct=0.828084 moved_to_echo=22587 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=19 rerouted_tokens_per_expert_current_ep_rank=[626, 719, 273, 122, 61, 63, 1159, 1771, 134, 0, 0, 0, 0, 0, 0, 0, 562, 272, 2707, 592, 1890, 0, 0, 1502, 0, 0, 0, 0, 0, 0, 0, 0, 261, 1792, 47, 1078, 2551, 164, 411, 1466, 0, 0, 0, 0, 0, 0, 0, 0, 655, 33, 1230, 352, 582, 706, 583, 121, 1193, 0, 0, 0, 0, 0, 0, 0, 629, 195, 808, 336, 60, 605, 503, 815, 0, 0, 0, 0, 0, 0, 0, 0, 1193, 729, 197, 94, 2025, 1317, 72, 2464, 272, 9, 0, 0, 0, 0, 0, 0, 1319, 114, 173, 165, 30, 124, 98, 613, 1407, 211, 0, 0, 0, 0, 0, 0, 1335, 1168, 148, 232, 81, 71, 500, 659, 2541, 162, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4928, 7525, 7770, 5455, 3951, 8372, 4254, 6897] avg=6144.0000 max=8372 max_rank=5 min=3951 min_rank=4 imbalance_ratio=1.362630 max_load_reduction=40780 max_load_reduction_pct=0.829671 moved_to_echo=23474 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=19 rerouted_tokens_per_expert_current_ep_rank=[498, 2405, 168, 213, 527, 197, 1980, 1841, 472, 0, 0, 0, 0, 0, 0, 0, 366, 935, 430, 1068, 1154, 0, 0, 1553, 0, 0, 0, 0, 0, 0, 0, 0, 53, 543, 157, 524, 943, 221, 610, 289, 0, 0, 0, 0, 0, 0, 0, 0, 454, 650, 3069, 200, 846, 68, 2263, 272, 1183, 0, 0, 0, 0, 0, 0, 0, 1279, 9, 343, 1365, 138, 1143, 320, 279, 0, 0, 0, 0, 0, 0, 0, 0, 38, 463, 401, 197, 709, 320, 1064, 270, 179, 36, 0, 0, 0, 0, 0, 0, 1702, 494, 1141, 494, 149, 106, 73, 1724, 431, 208, 0, 0, 0, 0, 0, 0, 1253, 1431, 1306, 196, 13, 360, 317, 302, 2587, 160, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8301, 5506, 3340, 9005, 4876, 3677, 6522, 7925] avg=6144.0000 max=9005 max_rank=3 min=3340 min_rank=2 imbalance_ratio=1.465658 max_load_reduction=40147 max_load_reduction_pct=0.816793 moved_to_echo=23000 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=19 rerouted_tokens_per_expert_current_ep_rank=[864, 1466, 189, 950, 41, 38, 1822, 2309, 515, 0, 0, 0, 0, 0, 0, 0, 418, 1040, 1019, 174, 860, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 80, 1577, 72, 105, 1923, 854, 740, 2223, 0, 0, 0, 0, 0, 0, 0, 0, 2419, 525, 608, 229, 951, 288, 165, 273, 988, 0, 0, 0, 0, 0, 0, 0, 759, 346, 457, 1212, 847, 230, 1519, 401, 0, 0, 0, 0, 0, 0, 0, 0, 164, 611, 192, 272, 2644, 894, 687, 2224, 824, 35, 0, 0, 0, 0, 0, 0, 230, 54, 635, 313, 123, 158, 8, 773, 1238, 176, 0, 0, 0, 0, 0, 0, 1435, 753, 195, 298, 12, 46, 47, 1003, 1303, 134, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8194, 3686, 7574, 6446, 5771, 8547, 3708, 5226] avg=6144.0000 max=8547 max_rank=5 min=3686 min_rank=1 imbalance_ratio=1.391113 max_load_reduction=40605 max_load_reduction_pct=0.826111 moved_to_echo=23252 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=19 rerouted_tokens_per_expert_current_ep_rank=[449, 803, 72, 266, 43, 23, 878, 870, 920, 0, 0, 0, 0, 0, 0, 0, 179, 1855, 1939, 1631, 1304, 0, 0, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 160, 2218, 71, 607, 1073, 175, 321, 472, 0, 0, 0, 0, 0, 0, 0, 0, 719, 1154, 529, 157, 227, 355, 220, 313, 753, 0, 0, 0, 0, 0, 0, 0, 977, 551, 821, 784, 181, 705, 2360, 1296, 0, 0, 0, 0, 0, 0, 0, 0, 378, 37, 331, 265, 934, 976, 595, 1318, 1282, 64, 0, 0, 0, 0, 0, 0, 1564, 39, 2054, 1733, 67, 153, 17, 1546, 1742, 132, 0, 0, 0, 0, 0, 0, 342, 678, 564, 130, 12, 24, 208, 10, 2403, 104, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4324, 7927, 5097, 4427, 7675, 6180, 9047, 4475] avg=6144.0000 max=9047 max_rank=6 min=4324 min_rank=0 imbalance_ratio=1.472493 max_load_reduction=40105 max_load_reduction_pct=0.815938 moved_to_echo=27377 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=19 rerouted_tokens_per_expert_current_ep_rank=[428, 610, 132, 73, 1049, 82, 195, 2150, 443, 0, 0, 0, 0, 0, 0, 0, 534, 896, 875, 252, 508, 0, 0, 715, 0, 0, 0, 0, 0, 0, 0, 0, 110, 508, 259, 647, 1625, 902, 964, 1559, 0, 0, 0, 0, 0, 0, 0, 0, 689, 219, 1924, 393, 481, 57, 137, 615, 1186, 0, 0, 0, 0, 0, 0, 0, 192, 480, 482, 1226, 710, 1263, 2225, 1128, 0, 0, 0, 0, 0, 0, 0, 0, 480, 284, 536, 678, 2396, 418, 41, 1549, 1209, 30, 0, 0, 0, 0, 0, 0, 916, 27, 780, 800, 473, 115, 147, 1196, 399, 209, 0, 0, 0, 0, 0, 0, 2838, 588, 1151, 111, 87, 291, 304, 278, 1735, 163, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5162, 3780, 6574, 5701, 7706, 7621, 5062, 7546] avg=6144.0000 max=7706 max_rank=4 min=3780 min_rank=1 imbalance_ratio=1.254232 max_load_reduction=41446 max_load_reduction_pct=0.843221 moved_to_echo=27935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=19 rerouted_tokens_per_expert_current_ep_rank=[696, 509, 262, 133, 893, 199, 450, 359, 1265, 0, 0, 0, 0, 0, 0, 0, 88, 2550, 314, 289, 1234, 0, 0, 2538, 0, 0, 0, 0, 0, 0, 0, 0, 202, 1641, 415, 490, 2300, 113, 620, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 1222, 983, 1930, 317, 246, 147, 67, 254, 1153, 0, 0, 0, 0, 0, 0, 0, 829, 423, 1302, 1373, 301, 1063, 300, 1597, 0, 0, 0, 0, 0, 0, 0, 0, 479, 42, 106, 716, 1126, 1145, 94, 1159, 163, 87, 0, 0, 0, 0, 0, 0, 835, 9, 2604, 768, 417, 955, 15, 748, 1289, 204, 0, 0, 0, 0, 0, 0, 368, 836, 207, 67, 49, 97, 266, 520, 391, 156, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4766, 7013, 7948, 6319, 7188, 5117, 7844, 2957] avg=6144.0000 max=7948 max_rank=2 min=2957 min_rank=7 imbalance_ratio=1.293620 max_load_reduction=41204 max_load_reduction_pct=0.838298 moved_to_echo=23106 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 3072, 2816, 2560, 12928, 6272, 3328, 10240, 4864, 384, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[8448, 1024, 9600, 6656, 2432, 2560, 1024, 8192, 7808, 1920, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5120, 10880, 1408, 2944, 3328, 1024, 8832, 11776, 4480, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[9600, 6528, 4480, 2176, 512, 1408, 2688, 4352, 16512, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[1408, 9984, 1664, 3712, 13440, 2816, 4736, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 8832, 10112, 5888, 9472, 0, 0, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6784, 2688, 6400, 8320, 2816, 5888, 8960, 7808, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7424, 4480, 11648, 2432, 3840, 2688, 3840, 2560, 10880, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=20 tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 2595, 1650, 369, 215, 258, 14, 299, 397, 44, 15, 139, 712, 1252, 297, 433, 290, 1631, 254, 275, 796, 1429, 1124, 934, 318, 886, 427, 111, 906, 786, 144, 30, 183, 76, 438, 2310, 1135, 246, 351, 125, 390, 1018, 15, 183, 1132, 702, 65, 3230, 609, 674, 389, 4366, 2166, 73, 932, 1630, 1147, 447, 848, 2246, 201] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=20 tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2854, 1143, 194, 72, 598, 212, 30, 815, 1156, 287, 3134, 3325, 729, 188, 399, 173, 1409, 335, 370, 1931, 408, 1779, 920, 1679, 265, 452, 195, 879, 306, 221, 20, 334, 98, 285, 1447, 46, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1003, 1013, 1339, 1291, 744, 1746, 233, 1303, 80, 403, 1209, 249, 74, 196, 413, 654] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=20 tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1613, 531, 441, 48, 653, 95, 179, 596, 371, 101, 236, 3594, 716, 223, 1751, 101, 566, 155, 2173, 290, 203, 1539, 940, 1682, 755, 857, 21, 335, 560, 440, 894, 91, 108, 273, 718, 1086, 383, 709, 207, 560, 383, 1162, 72, 291, 953, 117, 929, 1300, 390, 1235, 1815, 2106, 870, 21, 4511, 371, 202, 426, 1704, 337] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=20 tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1564, 1182, 314, 128, 109, 874, 287, 561, 1249, 528, 308, 1401, 852, 164, 857, 447, 433, 74, 287, 194, 45, 2239, 283, 1457, 347, 281, 126, 470, 650, 1460, 261, 273, 333, 295, 669, 901, 123, 425, 137, 785, 1329, 1517, 235, 214, 1757, 191, 1591, 1554, 451, 757, 1031, 631, 1325, 1461, 2253, 1321, 341, 1053, 1946, 481] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=20 tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 3395, 89, 65, 47, 118, 27, 29, 573, 1793, 44, 191, 1224, 221, 854, 532, 619, 1380, 461, 589, 3893, 395, 2345, 1566, 1199, 182, 144, 657, 1786, 562, 123, 248, 115, 519, 667, 1967, 479, 118, 1318, 350, 160, 614, 395, 365, 538, 1994, 26, 1243, 1866, 897, 577, 441, 1568, 297, 491, 1180, 992, 258, 1496, 1324, 321] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=20 tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 484, 2421, 963, 280, 50, 243, 104, 330, 623, 291, 133, 2020, 142, 990, 3680, 274, 398, 56, 108, 205, 355, 319, 1511, 1166, 338, 169, 45, 234, 1464, 262, 380, 383, 416, 1012, 658, 2662, 115, 720, 361, 17, 540, 122, 723, 946, 666, 371, 2089, 252, 221, 3065, 2983, 1175, 251, 1655, 1546, 41, 1538, 587, 196, 893] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=20 tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1987, 2297, 239, 110, 373, 269, 201, 343, 2186, 62, 22, 2797, 470, 991, 734, 259, 443, 21, 273, 801, 890, 2107, 1382, 1494, 28, 442, 206, 277, 418, 122, 244, 339, 82, 384, 2318, 2858, 457, 602, 86, 6, 1163, 259, 19, 46, 873, 54, 3138, 489, 314, 1177, 729, 1352, 25, 542, 2015, 338, 61, 272, 1311, 142] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=20 tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 2004, 312, 187, 199, 646, 30, 51, 697, 767, 152, 1380, 1157, 955, 340, 263, 266, 3349, 647, 446, 1087, 76, 1662, 2854, 2959, 1796, 5, 164, 32, 764, 48, 12, 549, 603, 2224, 335, 542, 47, 685, 299, 419, 480, 217, 226, 1065, 2092, 419, 1341, 1049, 608, 1488, 2103, 1282, 1171, 93, 563, 274, 372, 656, 549, 154] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2414, 228, 82, 71, 1917, 1650, 369, 215, 0, 0, 0, 0, 0, 0, 0, 0, 258, 14, 299, 397, 44, 15, 139, 712, 678, 209, 284, 0, 0, 0, 0, 0, 1252, 297, 433, 290, 1631, 254, 275, 796, 205, 127, 0, 0, 0, 0, 0, 0, 1429, 788, 934, 318, 886, 427, 111, 906, 0, 0, 0, 0, 0, 0, 0, 0, 786, 144, 30, 183, 76, 438, 2310, 1135, 2715, 0, 0, 0, 0, 0, 0, 0, 246, 351, 125, 390, 1018, 15, 183, 1132, 3230, 687, 0, 0, 0, 0, 0, 0, 702, 65, 0, 609, 674, 389, 680, 2166, 0, 0, 0, 0, 0, 0, 0, 0, 73, 932, 1425, 1147, 447, 848, 2246, 201, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6946, 3049, 5560, 5799, 7817, 7377, 5285, 7319] avg=6144.0000 max=7817 max_rank=4 min=3049 min_rank=1 imbalance_ratio=1.272298 max_load_reduction=41335 max_load_reduction_pct=0.840963 moved_to_echo=27798 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=20 rerouted_tokens_per_expert_current_ep_rank=[537, 263, 2, 423, 2507, 89, 65, 47, 0, 0, 0, 0, 0, 0, 0, 0, 118, 27, 29, 573, 1793, 44, 191, 1224, 888, 436, 28, 0, 0, 0, 0, 0, 221, 854, 532, 619, 1380, 461, 589, 3893, 148, 265, 0, 0, 0, 0, 0, 0, 395, 1644, 1566, 1199, 182, 144, 657, 1786, 0, 0, 0, 0, 0, 0, 0, 0, 562, 123, 248, 115, 519, 667, 1967, 479, 274, 0, 0, 0, 0, 0, 0, 0, 118, 1318, 350, 160, 614, 395, 365, 538, 1243, 69, 0, 0, 0, 0, 0, 0, 1994, 26, 0, 1866, 897, 577, 70, 1568, 0, 0, 0, 0, 0, 0, 0, 0, 297, 491, 1032, 992, 258, 1496, 1324, 321, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3933, 5351, 8962, 7573, 4954, 5170, 6998, 6211] avg=6144.0000 max=8962 max_rank=2 min=3933 min_rank=0 imbalance_ratio=1.458659 max_load_reduction=40190 max_load_reduction_pct=0.817668 moved_to_echo=23333 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2069, 28, 16, 1050, 1192, 531, 441, 48, 0, 0, 0, 0, 0, 0, 0, 0, 653, 95, 179, 596, 371, 101, 236, 3594, 421, 286, 118, 0, 0, 0, 0, 0, 716, 223, 1751, 101, 566, 155, 2173, 290, 567, 174, 0, 0, 0, 0, 0, 0, 203, 1079, 940, 1682, 755, 857, 21, 335, 0, 0, 0, 0, 0, 0, 0, 0, 560, 440, 894, 91, 108, 273, 718, 1086, 1128, 0, 0, 0, 0, 0, 0, 0, 383, 709, 207, 560, 383, 1162, 72, 291, 929, 285, 0, 0, 0, 0, 0, 0, 953, 117, 0, 1300, 390, 1235, 284, 2106, 0, 0, 0, 0, 0, 0, 0, 0, 870, 21, 3944, 371, 202, 426, 1704, 337, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5375, 6650, 6716, 5872, 5298, 4981, 6385, 7875] avg=6144.0000 max=7875 max_rank=7 min=4981 min_rank=5 imbalance_ratio=1.281738 max_load_reduction=41277 max_load_reduction_pct=0.839783 moved_to_echo=24539 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=20 rerouted_tokens_per_expert_current_ep_rank=[781, 97, 49, 962, 2108, 1143, 194, 72, 0, 0, 0, 0, 0, 0, 0, 0, 598, 212, 30, 815, 1156, 287, 3134, 3325, 746, 331, 15, 0, 0, 0, 0, 0, 729, 188, 399, 173, 1409, 335, 370, 1931, 152, 201, 0, 0, 0, 0, 0, 0, 408, 1247, 920, 1679, 265, 452, 195, 879, 0, 0, 0, 0, 0, 0, 0, 0, 306, 221, 20, 334, 98, 285, 1447, 46, 144, 0, 0, 0, 0, 0, 0, 0, 5, 2731, 479, 109, 1255, 225, 424, 1397, 1339, 36, 0, 0, 0, 0, 0, 0, 1003, 1013, 0, 1291, 744, 1746, 38, 1303, 0, 0, 0, 0, 0, 0, 0, 0, 80, 403, 1057, 249, 74, 196, 413, 654, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5406, 10649, 5887, 6045, 2901, 8000, 7138, 3126] avg=6144.0000 max=10649 max_rank=1 min=2901 min_rank=4 imbalance_ratio=1.733236 max_load_reduction=38503 max_load_reduction_pct=0.783346 moved_to_echo=21165 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=20 rerouted_tokens_per_expert_current_ep_rank=[488, 497, 242, 713, 1476, 312, 187, 199, 0, 0, 0, 0, 0, 0, 0, 0, 646, 30, 51, 697, 767, 152, 1380, 1157, 528, 313, 139, 0, 0, 0, 0, 0, 955, 340, 263, 266, 3349, 647, 446, 1087, 75, 189, 0, 0, 0, 0, 0, 0, 76, 1160, 2854, 2959, 1796, 5, 164, 32, 0, 0, 0, 0, 0, 0, 0, 0, 764, 48, 12, 549, 603, 2224, 335, 542, 1312, 0, 0, 0, 0, 0, 0, 0, 47, 685, 299, 419, 480, 217, 226, 1065, 1341, 335, 0, 0, 0, 0, 0, 0, 2092, 419, 0, 1049, 608, 1488, 317, 1282, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 93, 488, 274, 372, 656, 549, 154, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4114, 5860, 7617, 9046, 6389, 5114, 7255, 3757] avg=6144.0000 max=9046 max_rank=3 min=3757 min_rank=7 imbalance_ratio=1.472331 max_load_reduction=40106 max_load_reduction_pct=0.815959 moved_to_echo=22515 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=20 rerouted_tokens_per_expert_current_ep_rank=[1834, 414, 97, 565, 358, 2421, 963, 280, 0, 0, 0, 0, 0, 0, 0, 0, 50, 243, 104, 330, 623, 291, 133, 2020, 126, 59, 194, 0, 0, 0, 0, 0, 142, 990, 3680, 274, 398, 56, 108, 205, 194, 36, 0, 0, 0, 0, 0, 0, 355, 224, 1511, 1166, 338, 169, 45, 234, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 262, 380, 383, 416, 1012, 658, 2662, 1855, 0, 0, 0, 0, 0, 0, 0, 115, 720, 361, 17, 540, 122, 723, 946, 2089, 469, 0, 0, 0, 0, 0, 0, 666, 371, 0, 252, 221, 3065, 465, 1175, 0, 0, 0, 0, 0, 0, 0, 0, 251, 1655, 1352, 41, 1538, 587, 196, 893, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6932, 4173, 6083, 4042, 9092, 6102, 6215, 6513] avg=6144.0000 max=9092 max_rank=4 min=4042 min_rank=3 imbalance_ratio=1.479818 max_load_reduction=40060 max_load_reduction_pct=0.815023 moved_to_echo=27922 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=20 rerouted_tokens_per_expert_current_ep_rank=[2074, 483, 687, 1096, 1155, 1182, 314, 128, 0, 0, 0, 0, 0, 0, 0, 0, 109, 874, 287, 561, 1249, 528, 308, 1401, 409, 416, 67, 0, 0, 0, 0, 0, 852, 164, 857, 447, 433, 74, 287, 194, 283, 253, 0, 0, 0, 0, 0, 0, 45, 1570, 283, 1457, 347, 281, 126, 470, 0, 0, 0, 0, 0, 0, 0, 0, 650, 1460, 261, 273, 333, 295, 669, 901, 641, 0, 0, 0, 0, 0, 0, 0, 123, 425, 137, 785, 1329, 1517, 235, 214, 1591, 162, 0, 0, 0, 0, 0, 0, 1757, 191, 0, 1554, 451, 757, 161, 631, 0, 0, 0, 0, 0, 0, 0, 0, 1325, 1461, 1970, 1321, 341, 1053, 1946, 481, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7119, 6209, 3844, 4579, 5483, 6518, 5502, 9898] avg=6144.0000 max=9898 max_rank=7 min=3844 min_rank=2 imbalance_ratio=1.611003 max_load_reduction=39254 max_load_reduction_pct=0.798625 moved_to_echo=27401 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=20 rerouted_tokens_per_expert_current_ep_rank=[3343, 741, 7, 1122, 1468, 2297, 239, 110, 0, 0, 0, 0, 0, 0, 0, 0, 373, 269, 201, 343, 2186, 62, 22, 2797, 519, 392, 47, 0, 0, 0, 0, 0, 470, 991, 734, 259, 443, 21, 273, 801, 253, 238, 0, 0, 0, 0, 0, 0, 890, 1477, 1382, 1494, 28, 442, 206, 277, 0, 0, 0, 0, 0, 0, 0, 0, 418, 122, 244, 339, 82, 384, 2318, 2858, 453, 0, 0, 0, 0, 0, 0, 0, 457, 602, 86, 6, 1163, 259, 19, 46, 3138, 114, 0, 0, 0, 0, 0, 0, 873, 54, 0, 489, 314, 1177, 115, 1352, 0, 0, 0, 0, 0, 0, 0, 0, 25, 542, 1762, 338, 61, 272, 1311, 142, 0, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[9327, 7211, 4483, 6196, 7218, 5890, 4374, 4453] avg=6144.0000 max=9327 max_rank=0 min=4374 min_rank=6 imbalance_ratio=1.518066 max_load_reduction=39825 max_load_reduction_pct=0.810242 moved_to_echo=21935 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2816, 1792, 1280, 4352, 8192, 1536, 5632, 16256, 4352, 2560, 896, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49408 observed_loss=-256 tokens_per_expert=[4096, 5632, 13056, 4736, 3328, 5632, 9728, 3200, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[5376, 4096, 8704, 2432, 9728, 2048, 4608, 9216, 1920, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[13568, 2816, 1280, 6016, 12288, 9728, 2816, 1152, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[1536, 7552, 2048, 2560, 6784, 3968, 2304, 5632, 14976, 2176, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[10112, 2304, 0, 8448, 4352, 10496, 2176, 11648, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3840, 9216, 10496, 12032, 4608, 2816, 1536, 4992, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5632, 2944, 2176, 2304, 2304, 5632, 10496, 9728, 8576, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49408 slack_slots=896 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=21 tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 535, 237, 939, 65, 18, 1438, 37, 527, 122, 157, 305, 547, 3841, 2137, 487, 355, 455, 2376, 206, 161, 6311, 275, 9, 2527, 7, 1680, 12, 94, 17, 15, 844, 90, 411, 204, 1021, 363, 1936, 597, 542, 496, 1410, 974, 415, 340, 1675, 742, 139, 59, 1216, 198, 794, 200, 302, 84, 253, 1317] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=21 tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 65, 868, 1894, 109, 27, 465, 47, 834, 1337, 30, 269, 31, 2047, 1066, 169, 772, 309, 1080, 498, 185, 6768, 159, 29, 1019, 21, 337, 98, 52, 682, 59, 1377, 113, 1720, 815, 94, 37, 934, 76, 1413, 505, 664, 304, 2936, 153, 348, 2033, 384, 28, 281, 202, 2018, 79, 715, 203, 2426, 160] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=21 tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 512, 1038, 3057, 1010, 39, 174, 359, 600, 776, 49, 620, 299, 2205, 847, 152, 973, 105, 743, 549, 270, 5273, 98, 59, 149, 23, 615, 320, 11, 1233, 250, 1297, 116, 4291, 1560, 977, 582, 406, 390, 1143, 2443, 835, 301, 777, 136, 97, 1141, 185, 16, 215, 163, 2191, 908, 264, 1064, 1174, 8] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=21 tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 410, 2912, 5039, 291, 10, 384, 174, 628, 422, 39, 248, 65, 4267, 744, 537, 798, 322, 1425, 143, 696, 1922, 135, 57, 516, 5, 169, 1382, 17, 848, 107, 891, 76, 2653, 225, 779, 426, 402, 266, 1095, 3625, 86, 849, 1224, 246, 767, 2910, 751, 40, 466, 835, 894, 323, 49, 431, 154, 576] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=21 tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 628, 854, 3026, 433, 125, 871, 159, 479, 1063, 30, 600, 811, 1949, 385, 888, 616, 712, 2781, 989, 81, 4117, 205, 46, 461, 28, 673, 192, 322, 865, 361, 261, 334, 3258, 1983, 83, 572, 351, 141, 1874, 164, 1917, 476, 1235, 595, 334, 1795, 480, 20, 245, 71, 2439, 134, 96, 683, 1752, 129] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=21 tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 336, 2327, 1052, 151, 60, 1278, 57, 500, 2143, 234, 693, 489, 2555, 2088, 241, 526, 149, 924, 127, 102, 3622, 804, 107, 259, 13, 2882, 323, 67, 769, 427, 1026, 605, 2735, 961, 238, 122, 2101, 293, 1013, 1077, 3080, 227, 823, 148, 613, 1034, 254, 28, 59, 22, 460, 84, 570, 494, 1425, 661] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=21 tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 24, 323, 580, 554, 214, 309, 30, 82, 1885, 123, 491, 1230, 4280, 863, 661, 1130, 32, 921, 1427, 786, 1472, 360, 28, 10, 23, 234, 88, 513, 460, 29, 384, 993, 2139, 1244, 582, 236, 688, 268, 630, 923, 2162, 2647, 2658, 825, 94, 1553, 53, 86, 920, 575, 909, 75, 4, 286, 2220, 181] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=21 tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 329, 497, 3001, 689, 91, 456, 14, 180, 1220, 306, 182, 90, 1206, 974, 550, 422, 11, 3359, 793, 297, 3237, 309, 22, 97, 66, 1761, 28, 29, 535, 163, 1088, 706, 3698, 1789, 482, 21, 388, 102, 2098, 1736, 1782, 329, 1496, 389, 166, 800, 894, 428, 1236, 204, 1101, 7, 239, 986, 983, 449] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=21 rerouted_tokens_per_expert_current_ep_rank=[1697, 538, 45, 2209, 594, 97, 172, 1286, 1206, 434, 0, 0, 0, 0, 0, 0, 535, 237, 939, 65, 18, 1438, 37, 527, 199, 252, 0, 0, 0, 0, 0, 0, 122, 157, 305, 547, 2383, 2137, 487, 355, 0, 0, 0, 0, 0, 0, 0, 0, 455, 2376, 206, 161, 3550, 275, 9, 2527, 0, 0, 0, 0, 0, 0, 0, 0, 7, 1680, 12, 94, 17, 15, 844, 90, 339, 536, 0, 0, 0, 0, 0, 0, 72, 204, 1021, 363, 1936, 597, 542, 496, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 974, 415, 340, 1675, 543, 139, 59, 0, 0, 0, 0, 0, 0, 0, 0, 1216, 198, 794, 200, 302, 84, 253, 1317, 1791, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8278, 4247, 6493, 9559, 3634, 5231, 5555, 6155] avg=6144.0000 max=9559 max_rank=3 min=3634 min_rank=4 imbalance_ratio=1.555827 max_load_reduction=39593 max_load_reduction_pct=0.805522 moved_to_echo=20575 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=21 rerouted_tokens_per_expert_current_ep_rank=[169, 328, 638, 756, 81, 34, 643, 331, 611, 283, 0, 0, 0, 0, 0, 0, 628, 854, 3026, 433, 125, 871, 159, 479, 483, 127, 0, 0, 0, 0, 0, 0, 1063, 30, 600, 811, 1211, 385, 888, 616, 0, 0, 0, 0, 0, 0, 0, 0, 712, 2781, 989, 81, 2317, 205, 46, 461, 0, 0, 0, 0, 0, 0, 0, 0, 28, 673, 192, 322, 865, 361, 261, 334, 2690, 349, 0, 0, 0, 0, 0, 0, 568, 1983, 83, 572, 351, 141, 1874, 164, 0, 0, 0, 0, 0, 0, 0, 0, 1917, 476, 1235, 595, 334, 1312, 480, 20, 0, 0, 0, 0, 0, 0, 0, 0, 245, 71, 2439, 134, 96, 683, 1752, 129, 1168, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3874, 7185, 5604, 7592, 6075, 5736, 6369, 6717] avg=6144.0000 max=7592 max_rank=3 min=3874 min_rank=0 imbalance_ratio=1.235677 max_load_reduction=41560 max_load_reduction_pct=0.845540 moved_to_echo=24897 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=21 rerouted_tokens_per_expert_current_ep_rank=[236, 308, 78, 950, 102, 167, 696, 864, 1344, 136, 0, 0, 0, 0, 0, 0, 410, 2912, 5039, 291, 10, 384, 174, 628, 787, 284, 0, 0, 0, 0, 0, 0, 422, 39, 248, 65, 2639, 744, 537, 798, 0, 0, 0, 0, 0, 0, 0, 0, 322, 1425, 143, 696, 1071, 135, 57, 516, 0, 0, 0, 0, 0, 0, 0, 0, 5, 169, 1382, 17, 848, 107, 891, 76, 2195, 166, 0, 0, 0, 0, 0, 0, 458, 225, 779, 426, 402, 266, 1095, 3625, 0, 0, 0, 0, 0, 0, 0, 0, 86, 849, 1224, 246, 767, 2123, 751, 40, 0, 0, 0, 0, 0, 0, 0, 0, 466, 835, 894, 323, 49, 431, 154, 576, 549, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4881, 10919, 5492, 4365, 5856, 7276, 6086, 4277] avg=6144.0000 max=10919 max_rank=1 min=4277 min_rank=7 imbalance_ratio=1.777181 max_load_reduction=38233 max_load_reduction_pct=0.777852 moved_to_echo=23495 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=21 rerouted_tokens_per_expert_current_ep_rank=[317, 515, 53, 780, 32, 50, 511, 1806, 692, 363, 0, 0, 0, 0, 0, 0, 512, 1038, 3057, 1010, 39, 174, 359, 600, 307, 144, 0, 0, 0, 0, 0, 0, 776, 49, 620, 299, 1369, 847, 152, 973, 0, 0, 0, 0, 0, 0, 0, 0, 105, 743, 549, 270, 2965, 98, 59, 149, 0, 0, 0, 0, 0, 0, 0, 0, 23, 615, 320, 11, 1233, 250, 1297, 116, 3543, 448, 0, 0, 0, 0, 0, 0, 748, 1560, 977, 582, 406, 390, 1143, 2443, 0, 0, 0, 0, 0, 0, 0, 0, 835, 301, 777, 136, 97, 834, 185, 16, 0, 0, 0, 0, 0, 0, 0, 0, 215, 163, 2191, 908, 264, 1064, 1174, 8, 1497, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5119, 7240, 5085, 4938, 7856, 8249, 3181, 7484] avg=6144.0000 max=8249 max_rank=5 min=3181 min_rank=6 imbalance_ratio=1.342611 max_load_reduction=40903 max_load_reduction_pct=0.832174 moved_to_echo=26770 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=21 rerouted_tokens_per_expert_current_ep_rank=[42, 661, 1115, 688, 262, 458, 787, 3795, 642, 465, 0, 0, 0, 0, 0, 0, 65, 868, 1894, 109, 27, 465, 47, 834, 547, 134, 0, 0, 0, 0, 0, 0, 1337, 30, 269, 31, 1271, 1066, 169, 772, 0, 0, 0, 0, 0, 0, 0, 0, 309, 1080, 498, 185, 3807, 159, 29, 1019, 0, 0, 0, 0, 0, 0, 0, 0, 21, 337, 98, 52, 682, 59, 1377, 113, 1420, 575, 0, 0, 0, 0, 0, 0, 300, 815, 94, 37, 934, 76, 1413, 505, 0, 0, 0, 0, 0, 0, 0, 0, 664, 304, 2936, 153, 348, 1486, 384, 28, 0, 0, 0, 0, 0, 0, 0, 0, 281, 202, 2018, 79, 715, 203, 2426, 160, 1921, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8915, 4990, 4945, 7086, 4734, 4174, 6303, 8005] avg=6144.0000 max=8915 max_rank=0 min=4174 min_rank=5 imbalance_ratio=1.451009 max_load_reduction=40237 max_load_reduction_pct=0.818624 moved_to_echo=23216 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=21 rerouted_tokens_per_expert_current_ep_rank=[567, 45, 1741, 232, 267, 101, 195, 546, 802, 249, 0, 0, 0, 0, 0, 0, 336, 2327, 1052, 151, 60, 1278, 57, 500, 278, 167, 0, 0, 0, 0, 0, 0, 2143, 234, 693, 489, 1586, 2088, 241, 526, 0, 0, 0, 0, 0, 0, 0, 0, 149, 924, 127, 102, 2038, 804, 107, 259, 0, 0, 0, 0, 0, 0, 0, 0, 13, 2882, 323, 67, 769, 427, 1026, 605, 2258, 307, 0, 0, 0, 0, 0, 0, 477, 961, 238, 122, 2101, 293, 1013, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 3080, 227, 823, 148, 613, 756, 254, 28, 0, 0, 0, 0, 0, 0, 0, 0, 59, 22, 460, 84, 570, 494, 1425, 661, 1028, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4745, 6206, 8000, 4510, 8677, 6282, 5929, 4803] avg=6144.0000 max=8677 max_rank=4 min=4510 min_rank=3 imbalance_ratio=1.412272 max_load_reduction=40475 max_load_reduction_pct=0.823466 moved_to_echo=25691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=21 rerouted_tokens_per_expert_current_ep_rank=[354, 204, 413, 94, 438, 256, 1375, 1507, 378, 222, 0, 0, 0, 0, 0, 0, 329, 497, 3001, 689, 91, 456, 14, 180, 215, 79, 0, 0, 0, 0, 0, 0, 1220, 306, 182, 90, 749, 974, 550, 422, 0, 0, 0, 0, 0, 0, 0, 0, 11, 3359, 793, 297, 1821, 309, 22, 97, 0, 0, 0, 0, 0, 0, 0, 0, 66, 1761, 28, 29, 535, 163, 1088, 706, 3054, 275, 0, 0, 0, 0, 0, 0, 644, 1789, 482, 21, 388, 102, 2098, 1736, 0, 0, 0, 0, 0, 0, 0, 0, 1782, 329, 1496, 389, 166, 585, 894, 428, 0, 0, 0, 0, 0, 0, 0, 0, 1236, 204, 1101, 7, 239, 986, 983, 449, 919, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5241, 5551, 4493, 6709, 7705, 7260, 6069, 6124] avg=6144.0000 max=7705 max_rank=4 min=4493 min_rank=2 imbalance_ratio=1.254069 max_load_reduction=41447 max_load_reduction_pct=0.843241 moved_to_echo=27158 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=21 rerouted_tokens_per_expert_current_ep_rank=[2605, 649, 514, 351, 102, 142, 199, 2093, 1343, 101, 0, 0, 0, 0, 0, 0, 24, 323, 580, 554, 214, 309, 30, 82, 418, 280, 0, 0, 0, 0, 0, 0, 1885, 123, 491, 1230, 2657, 863, 661, 1130, 0, 0, 0, 0, 0, 0, 0, 0, 32, 921, 1427, 786, 829, 360, 28, 10, 0, 0, 0, 0, 0, 0, 0, 0, 23, 234, 88, 513, 460, 29, 384, 993, 1766, 125, 0, 0, 0, 0, 0, 0, 373, 1244, 582, 236, 688, 268, 630, 923, 0, 0, 0, 0, 0, 0, 0, 0, 2162, 2647, 2658, 825, 94, 1135, 53, 86, 0, 0, 0, 0, 0, 0, 0, 0, 920, 575, 909, 75, 4, 286, 2220, 181, 417, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8099, 2814, 9040, 4393, 4615, 4944, 9660, 5587] avg=6144.0000 max=9660 max_rank=6 min=2814 min_rank=1 imbalance_ratio=1.572266 max_load_reduction=39492 max_load_reduction_pct=0.803467 moved_to_echo=24806 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4736, 2304, 10880, 1920, 2304, 4352, 10496, 3584, 9344, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[3712, 8832, 4352, 2432, 7296, 2176, 9856, 11008, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9088, 1024, 3456, 3584, 13952, 9216, 3712, 5632, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2944, 9088, 18688, 3328, 640, 5376, 896, 3840, 3328, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[12032, 6144, 11648, 2944, 4096, 8832, 3200, 768, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[6016, 3328, 4608, 6144, 1920, 1408, 4608, 12288, 7040, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[256, 8448, 2560, 1152, 5504, 1536, 7168, 3072, 17280, 2816, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[2176, 13696, 4736, 2688, 18432, 2432, 384, 5120, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=22 tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 969, 127, 1426, 490, 1688, 615, 668, 586, 568, 367, 917, 488, 235, 1217, 2866, 1496, 416, 169, 306, 50, 1243, 1695, 7, 556, 30, 6703, 769, 146, 79, 273, 962, 1563, 1030, 1912, 63, 106, 201, 81, 756, 261, 147, 1147, 490, 624, 197, 829, 37, 102, 962, 1693, 219, 292, 1047, 571, 1297, 636] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=22 tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 3863, 0, 195, 570, 1120, 611, 801, 1419, 1521, 2114, 308, 1142, 107, 1242, 901, 1173, 250, 842, 4, 756, 128, 1739, 6, 605, 35, 4181, 759, 227, 198, 1935, 316, 483, 640, 369, 108, 19, 418, 2236, 168, 327, 169, 259, 47, 1154, 97, 1459, 42, 13, 582, 437, 790, 280, 37, 1181, 19, 92] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=22 tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 1891, 147, 77, 234, 999, 2417, 801, 558, 695, 1124, 857, 480, 126, 1097, 2524, 1416, 58, 1565, 305, 175, 1992, 1189, 79, 533, 115, 2537, 1198, 398, 176, 1150, 3301, 868, 1003, 1622, 57, 255, 33, 965, 388, 638, 87, 808, 434, 288, 348, 804, 446, 824, 1807, 605, 209, 206, 489, 374, 228, 551] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=22 tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1845, 69, 2276, 132, 79, 505, 17, 1443, 544, 2416, 3536, 98, 25, 592, 2713, 1604, 123, 875, 64, 29, 868, 1576, 5, 376, 107, 3575, 1757, 1022, 164, 496, 3840, 1266, 137, 252, 21, 31, 10, 113, 42, 199, 343, 1362, 39, 977, 44, 1192, 197, 69, 138, 1175, 222, 1567, 63, 1408, 311, 1318] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=22 tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 4657, 15, 705, 360, 202, 434, 381, 581, 854, 94, 1083, 536, 64, 1826, 3464, 653, 409, 2003, 11, 188, 385, 2706, 12, 696, 109, 3075, 1675, 553, 417, 1155, 1482, 614, 535, 172, 250, 1083, 19, 182, 549, 1141, 437, 1239, 729, 1048, 41, 1313, 879, 6, 263, 155, 346, 402, 692, 1483, 173, 1974] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=22 tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 2598, 414, 1291, 504, 363, 511, 2466, 372, 213, 278, 385, 311, 76, 2192, 4554, 24, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1, 1811, 189, 407, 500, 736, 245, 154, 654, 463, 432, 326, 1484, 390, 1320, 283, 1233, 1187, 534, 550, 204, 3437, 554, 34, 509, 230, 365, 478, 1121, 517, 127, 1418] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=22 tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 3937, 35, 444, 578, 36, 358, 1828, 2205, 36, 554, 548, 1032, 490, 1781, 4492, 2266, 441, 1682, 96, 664, 2140, 1424, 413, 215, 96, 3321, 116, 523, 178, 449, 622, 148, 336, 743, 424, 437, 49, 233, 67, 505, 956, 506, 251, 356, 21, 1341, 333, 3, 1636, 146, 161, 1790, 766, 869, 275, 246] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=22 tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 3426, 267, 25, 161, 26, 510, 1125, 994, 1177, 4735, 1405, 1273, 111, 4360, 4711, 249, 970, 1206, 0, 126, 1172, 766, 11, 167, 159, 295, 1357, 122, 631, 583, 2992, 224, 1048, 1354, 80, 242, 11, 276, 73, 560, 190, 905, 292, 342, 29, 277, 556, 19, 245, 34, 230, 210, 51, 1540, 95, 498] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=22 rerouted_tokens_per_expert_current_ep_rank=[106, 53, 381, 686, 549, 139, 2408, 440, 2470, 93, 0, 0, 0, 0, 0, 0, 497, 127, 1426, 490, 1688, 615, 668, 586, 0, 0, 0, 0, 0, 0, 0, 0, 568, 367, 917, 488, 235, 626, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 416, 169, 306, 50, 1243, 1695, 7, 556, 572, 0, 0, 0, 0, 0, 0, 0, 30, 2388, 769, 146, 79, 273, 962, 1563, 0, 0, 0, 0, 0, 0, 0, 0, 1030, 1912, 63, 106, 201, 81, 756, 261, 2073, 0, 0, 0, 0, 0, 0, 0, 147, 1147, 490, 624, 197, 829, 37, 102, 793, 1845, 0, 0, 0, 0, 0, 0, 962, 1693, 219, 292, 1047, 571, 1297, 636, 379, 19, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7325, 6097, 4697, 5014, 6210, 6483, 6211, 7115] avg=6144.0000 max=7325 max_rank=0 min=4697 min_rank=2 imbalance_ratio=1.192220 max_load_reduction=41827 max_load_reduction_pct=0.850972 moved_to_echo=26019 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=22 rerouted_tokens_per_expert_current_ep_rank=[558, 81, 38, 596, 40, 19, 1177, 2150, 108, 328, 0, 0, 0, 0, 0, 0, 1758, 267, 25, 161, 26, 510, 1125, 994, 0, 0, 0, 0, 0, 0, 0, 0, 1177, 4735, 1405, 1273, 111, 2240, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 970, 1206, 0, 126, 1172, 766, 11, 167, 2050, 0, 0, 0, 0, 0, 0, 0, 159, 106, 1357, 122, 631, 583, 2992, 224, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 1354, 80, 242, 11, 276, 73, 560, 3408, 0, 0, 0, 0, 0, 0, 0, 190, 905, 292, 342, 29, 277, 556, 19, 1303, 81, 0, 0, 0, 0, 0, 0, 245, 34, 230, 210, 51, 1540, 95, 498, 1340, 70, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5095, 4866, 11190, 6468, 6174, 7052, 3994, 4313] avg=6144.0000 max=11190 max_rank=2 min=3994 min_rank=6 imbalance_ratio=1.821289 max_load_reduction=37962 max_load_reduction_pct=0.772339 moved_to_echo=21533 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=22 rerouted_tokens_per_expert_current_ep_rank=[131, 82, 291, 1167, 199, 28, 669, 75, 1133, 447, 0, 0, 0, 0, 0, 0, 2389, 15, 705, 360, 202, 434, 381, 581, 0, 0, 0, 0, 0, 0, 0, 0, 854, 94, 1083, 536, 64, 939, 0, 653, 0, 0, 0, 0, 0, 0, 0, 0, 409, 2003, 11, 188, 385, 2706, 12, 696, 858, 0, 0, 0, 0, 0, 0, 0, 109, 1096, 1675, 553, 417, 1155, 1482, 614, 0, 0, 0, 0, 0, 0, 0, 0, 535, 172, 250, 1083, 19, 182, 549, 1141, 2505, 0, 0, 0, 0, 0, 0, 0, 437, 1239, 729, 1048, 41, 1313, 879, 6, 959, 846, 0, 0, 0, 0, 0, 0, 263, 155, 346, 402, 692, 1483, 173, 1974, 1821, 29, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4222, 5067, 4223, 7268, 7101, 6436, 7497, 7338] avg=6144.0000 max=7497 max_rank=6 min=4222 min_rank=0 imbalance_ratio=1.220215 max_load_reduction=41655 max_load_reduction_pct=0.847473 moved_to_echo=28372 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=22 rerouted_tokens_per_expert_current_ep_rank=[297, 231, 509, 406, 1335, 1636, 2667, 1577, 1540, 370, 0, 0, 0, 0, 0, 0, 1983, 0, 195, 570, 1120, 611, 801, 1419, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 2114, 308, 1142, 107, 639, 0, 1173, 0, 0, 0, 0, 0, 0, 0, 0, 250, 842, 4, 756, 128, 1739, 6, 605, 583, 0, 0, 0, 0, 0, 0, 0, 35, 1490, 759, 227, 198, 1935, 316, 483, 0, 0, 0, 0, 0, 0, 0, 0, 640, 369, 108, 19, 418, 2236, 168, 327, 652, 0, 0, 0, 0, 0, 0, 0, 169, 259, 47, 1154, 97, 1459, 42, 13, 249, 1151, 0, 0, 0, 0, 0, 0, 582, 437, 790, 280, 37, 1181, 19, 92, 1510, 20, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[10568, 6699, 7004, 4913, 5443, 4937, 4640, 4948] avg=6144.0000 max=10568 max_rank=0 min=4640 min_rank=6 imbalance_ratio=1.720052 max_load_reduction=38584 max_load_reduction_pct=0.784993 moved_to_echo=19968 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=22 rerouted_tokens_per_expert_current_ep_rank=[359, 315, 120, 631, 455, 46, 1024, 604, 1223, 377, 0, 0, 0, 0, 0, 0, 2021, 35, 444, 578, 36, 358, 1828, 2205, 0, 0, 0, 0, 0, 0, 0, 0, 36, 554, 548, 1032, 490, 916, 0, 2266, 0, 0, 0, 0, 0, 0, 0, 0, 441, 1682, 96, 664, 2140, 1424, 413, 215, 837, 0, 0, 0, 0, 0, 0, 0, 96, 1184, 116, 523, 178, 449, 622, 148, 0, 0, 0, 0, 0, 0, 0, 0, 336, 743, 424, 437, 49, 233, 67, 505, 3249, 0, 0, 0, 0, 0, 0, 0, 956, 506, 251, 356, 21, 1341, 333, 3, 1243, 914, 0, 0, 0, 0, 0, 0, 1636, 146, 161, 1790, 766, 869, 275, 246, 1539, 28, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5154, 7505, 5842, 7912, 3316, 6043, 5924, 7456] avg=6144.0000 max=7912 max_rank=3 min=3316 min_rank=4 imbalance_ratio=1.287760 max_load_reduction=41240 max_load_reduction_pct=0.839030 moved_to_echo=22739 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=22 rerouted_tokens_per_expert_current_ep_rank=[228, 109, 584, 536, 401, 487, 1613, 643, 934, 181, 0, 0, 0, 0, 0, 0, 971, 147, 77, 234, 999, 2417, 801, 558, 0, 0, 0, 0, 0, 0, 0, 0, 695, 1124, 857, 480, 126, 565, 0, 1416, 0, 0, 0, 0, 0, 0, 0, 0, 58, 1565, 305, 175, 1992, 1189, 79, 533, 515, 0, 0, 0, 0, 0, 0, 0, 115, 905, 1198, 398, 176, 1150, 3301, 868, 0, 0, 0, 0, 0, 0, 0, 0, 1003, 1622, 57, 255, 33, 965, 388, 638, 1825, 0, 0, 0, 0, 0, 0, 0, 87, 808, 434, 288, 348, 804, 446, 824, 699, 698, 0, 0, 0, 0, 0, 0, 1807, 605, 209, 206, 489, 374, 228, 551, 739, 17, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5716, 6204, 5263, 6411, 8111, 6786, 5436, 5225] avg=6144.0000 max=8111 max_rank=4 min=5225 min_rank=7 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=25558 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=22 rerouted_tokens_per_expert_current_ep_rank=[232, 136, 394, 787, 1588, 129, 1398, 105, 671, 253, 0, 0, 0, 0, 0, 0, 1325, 414, 1291, 504, 363, 511, 2466, 372, 0, 0, 0, 0, 0, 0, 0, 0, 213, 278, 385, 311, 76, 1118, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 570, 1796, 7, 338, 2034, 1018, 53, 122, 1034, 0, 0, 0, 0, 0, 0, 0, 1, 639, 189, 407, 500, 736, 245, 154, 0, 0, 0, 0, 0, 0, 0, 0, 654, 463, 432, 326, 1484, 390, 1320, 283, 3295, 0, 0, 0, 0, 0, 0, 0, 1233, 1187, 534, 550, 204, 3437, 554, 34, 1259, 501, 0, 0, 0, 0, 0, 0, 509, 230, 365, 478, 1121, 517, 127, 1418, 1020, 40, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5693, 7246, 2405, 6972, 2871, 8647, 9493, 5825] avg=6144.0000 max=9493 max_rank=6 min=2405 min_rank=2 imbalance_ratio=1.545085 max_load_reduction=39659 max_load_reduction_pct=0.806864 moved_to_echo=26836 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=22 rerouted_tokens_per_expert_current_ep_rank=[440, 332, 14, 133, 821, 56, 1924, 165, 1317, 177, 0, 0, 0, 0, 0, 0, 947, 69, 2276, 132, 79, 505, 17, 1443, 0, 0, 0, 0, 0, 0, 0, 0, 544, 2416, 3536, 98, 25, 305, 0, 1604, 0, 0, 0, 0, 0, 0, 0, 0, 123, 875, 64, 29, 868, 1576, 5, 376, 278, 0, 0, 0, 0, 0, 0, 0, 107, 1274, 1757, 1022, 164, 496, 3840, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 137, 252, 21, 31, 10, 113, 42, 199, 1963, 0, 0, 0, 0, 0, 0, 0, 343, 1362, 39, 977, 44, 1192, 197, 69, 750, 984, 0, 0, 0, 0, 0, 0, 138, 1175, 222, 1567, 63, 1408, 311, 1318, 721, 9, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5379, 5468, 8528, 4194, 9926, 2768, 5957, 6932] avg=6144.0000 max=9926 max_rank=4 min=2768 min_rank=5 imbalance_ratio=1.615560 max_load_reduction=39226 max_load_reduction_pct=0.798055 moved_to_echo=25583 active_clone_links=8 active_echo_slots_local=8 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[11904, 1152, 6528, 3072, 4608, 6016, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5632, 11776, 9088, 5376, 1280, 7424, 0, 8960, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[6144, 4480, 2560, 5248, 4352, 8064, 2560, 6784, 9088, 256, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[5504, 6912, 1536, 2560, 2304, 4480, 3456, 3968, 19072, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[3584, 7424, 2816, 5376, 1024, 10752, 3072, 1152, 7296, 7040, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2432, 1408, 2432, 4992, 5504, 2560, 12928, 5760, 9472, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[768, 9088, 7936, 3456, 2432, 6784, 13824, 5376, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3328, 10240, 896, 2432, 9984, 12160, 640, 3328, 6784, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=23 tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 639, 505, 203, 168, 2, 857, 41, 974, 1171, 676, 16, 658, 290, 1714, 4141, 503, 219, 425, 2, 2589, 907, 2159, 633, 290, 5, 1814, 609, 1030, 24, 67, 37, 155, 553, 1584, 114, 1056, 708, 971, 91, 596, 454, 164, 489, 50, 43, 553, 310, 778, 26, 705, 849, 3464, 202, 872, 13, 873, 247, 1412, 3562, 841] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=23 tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1765, 376, 255, 554, 147, 911, 522, 528, 1691, 1546, 104, 175, 379, 954, 1729, 866, 62, 139, 695, 518, 1250, 1494, 37, 1411, 283, 847, 511, 583, 984, 570, 1994, 926, 189, 246, 251, 174, 45, 78, 1946, 281, 2221, 432, 1102, 406, 10, 46, 31, 2856, 485, 1201, 2323, 1621, 77, 392, 95, 108, 729, 504, 721, 495] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=23 tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1686, 184, 178, 247, 157, 688, 141, 1660, 902, 175, 61, 415, 1208, 367, 680, 287, 420, 42, 37, 1528, 1266, 1875, 31, 496, 45, 2202, 363, 2808, 77, 39, 56, 1401, 56, 517, 211, 674, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 269, 725, 78, 2404, 9, 2736, 1875, 2406, 269, 162, 41, 309, 585, 662, 333, 2679] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=23 tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1914, 36, 100, 2189, 60, 550, 27, 319, 2226, 1602, 70, 438, 729, 1087, 4533, 2120, 632, 74, 505, 544, 641, 778, 593, 647, 195, 399, 1749, 2846, 90, 448, 645, 603, 47, 2003, 245, 27, 83, 95, 1477, 365, 476, 37, 120, 125, 15, 636, 97, 1621, 423, 642, 1076, 1065, 334, 979, 3, 741, 33, 152, 1132, 1163] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=23 tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 1073, 315, 351, 1048, 101, 78, 15, 1284, 357, 1959, 2, 1466, 2625, 2226, 616, 1153, 347, 213, 78, 991, 146, 1295, 109, 228, 16, 1748, 1305, 1418, 361, 1268, 108, 901, 106, 844, 549, 63, 26, 103, 256, 1971, 1550, 21, 483, 159, 68, 371, 67, 2193, 2, 2522, 773, 1100, 424, 983, 918, 111, 37, 1666, 2243, 340] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=23 tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1443, 75, 174, 744, 213, 647, 83, 1438, 1676, 2098, 20, 646, 63, 1500, 1721, 463, 890, 56, 159, 1292, 1150, 3088, 10, 1954, 56, 783, 136, 1056, 24, 411, 144, 688, 979, 638, 55, 50, 715, 111, 564, 1033, 1164, 740, 1867, 962, 107, 213, 633, 676, 750, 2548, 774, 1655, 133, 83, 39, 765, 48, 738, 2150, 846] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=23 tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 2341, 202, 57, 1415, 170, 1801, 41, 724, 1227, 1076, 752, 142, 764, 1683, 551, 546, 793, 65, 73, 4125, 958, 786, 440, 273, 86, 2112, 196, 1876, 196, 31, 687, 459, 0, 1777, 638, 1228, 1187, 189, 961, 86, 294, 513, 811, 582, 191, 749, 57, 371, 51, 1688, 207, 2952, 551, 348, 181, 293, 101, 403, 1423, 414] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=23 tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 847, 36, 78, 346, 22, 573, 72, 1043, 2324, 2092, 169, 193, 1024, 1632, 588, 863, 2818, 226, 4, 254, 1572, 874, 25, 252, 13, 2574, 495, 398, 982, 35, 56, 1260, 228, 735, 896, 264, 92, 519, 1024, 371, 812, 225, 1383, 620, 80, 772, 63, 275, 192, 1009, 1463, 2879, 200, 564, 10, 3898, 534, 551, 1211, 916] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1091, 256, 412, 2243, 808, 315, 351, 1048, 0, 0, 0, 0, 0, 0, 0, 0, 101, 78, 15, 1284, 357, 1959, 2, 1466, 265, 314, 0, 0, 0, 0, 0, 0, 2625, 2226, 140, 1153, 347, 213, 78, 991, 0, 0, 0, 0, 0, 0, 0, 0, 146, 1295, 109, 228, 16, 479, 1305, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 361, 1268, 108, 901, 106, 844, 549, 63, 476, 331, 0, 0, 0, 0, 0, 0, 26, 103, 256, 1971, 1550, 21, 483, 159, 289, 212, 0, 0, 0, 0, 0, 0, 68, 371, 67, 2193, 2, 2522, 773, 480, 0, 0, 0, 0, 0, 0, 0, 0, 424, 983, 918, 111, 37, 1666, 2243, 340, 743, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6524, 5841, 7773, 4996, 5007, 5070, 6476, 7465] avg=6144.0000 max=7773 max_rank=2 min=4996 min_rank=3 imbalance_ratio=1.265137 max_load_reduction=41379 max_load_reduction_pct=0.841858 moved_to_echo=24018 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1767, 713, 1359, 419, 1763, 202, 57, 1415, 0, 0, 0, 0, 0, 0, 0, 0, 170, 1801, 41, 724, 1227, 1076, 752, 142, 578, 379, 0, 0, 0, 0, 0, 0, 764, 1683, 125, 546, 793, 65, 73, 4125, 0, 0, 0, 0, 0, 0, 0, 0, 958, 786, 440, 273, 86, 578, 196, 1876, 0, 0, 0, 0, 0, 0, 0, 0, 196, 31, 687, 459, 0, 1777, 638, 1228, 426, 888, 0, 0, 0, 0, 0, 0, 1187, 189, 961, 86, 294, 513, 811, 582, 777, 257, 0, 0, 0, 0, 0, 0, 191, 749, 57, 371, 51, 1688, 207, 1287, 0, 0, 0, 0, 0, 0, 0, 0, 551, 348, 181, 293, 101, 403, 1423, 414, 898, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7695, 6890, 8174, 5193, 6330, 5657, 4601, 4612] avg=6144.0000 max=8174 max_rank=2 min=4601 min_rank=6 imbalance_ratio=1.330404 max_load_reduction=40978 max_load_reduction_pct=0.833700 moved_to_echo=21200 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=23 rerouted_tokens_per_expert_current_ep_rank=[153, 84, 3422, 892, 1441, 36, 100, 2189, 0, 0, 0, 0, 0, 0, 0, 0, 60, 550, 27, 319, 2226, 1602, 70, 438, 473, 71, 0, 0, 0, 0, 0, 0, 729, 1087, 1025, 2120, 632, 74, 505, 544, 0, 0, 0, 0, 0, 0, 0, 0, 641, 778, 593, 647, 195, 111, 1749, 2846, 0, 0, 0, 0, 0, 0, 0, 0, 90, 448, 645, 603, 47, 2003, 245, 27, 3508, 320, 0, 0, 0, 0, 0, 0, 83, 95, 1477, 365, 476, 37, 120, 125, 280, 48, 0, 0, 0, 0, 0, 0, 15, 636, 97, 1621, 423, 642, 1076, 465, 0, 0, 0, 0, 0, 0, 0, 0, 334, 979, 3, 741, 33, 152, 1132, 1163, 169, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8317, 5836, 6716, 7560, 7936, 3106, 4975, 4706] avg=6144.0000 max=8317 max_rank=0 min=3106 min_rank=5 imbalance_ratio=1.353678 max_load_reduction=40835 max_load_reduction_pct=0.830790 moved_to_echo=20723 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=23 rerouted_tokens_per_expert_current_ep_rank=[2384, 209, 2204, 484, 1325, 376, 255, 554, 0, 0, 0, 0, 0, 0, 0, 0, 147, 911, 522, 528, 1691, 1546, 104, 175, 440, 157, 0, 0, 0, 0, 0, 0, 379, 954, 387, 866, 62, 139, 695, 518, 0, 0, 0, 0, 0, 0, 0, 0, 1250, 1494, 37, 1411, 283, 219, 511, 583, 0, 0, 0, 0, 0, 0, 0, 0, 984, 570, 1994, 926, 189, 246, 251, 174, 1342, 491, 0, 0, 0, 0, 0, 0, 45, 78, 1946, 281, 2221, 432, 1102, 406, 432, 106, 0, 0, 0, 0, 0, 0, 10, 46, 31, 2856, 485, 1201, 2323, 698, 0, 0, 0, 0, 0, 0, 0, 0, 77, 392, 95, 108, 729, 504, 721, 495, 365, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7791, 6221, 4000, 5788, 7167, 7049, 7650, 3486] avg=6144.0000 max=7791 max_rank=0 min=3486 min_rank=7 imbalance_ratio=1.268066 max_load_reduction=41361 max_load_reduction_pct=0.841492 moved_to_echo=25352 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=23 rerouted_tokens_per_expert_current_ep_rank=[924, 281, 665, 2179, 482, 505, 203, 168, 0, 0, 0, 0, 0, 0, 0, 0, 2, 857, 41, 974, 1171, 676, 16, 658, 157, 326, 0, 0, 0, 0, 0, 0, 290, 1714, 937, 503, 219, 425, 2, 2589, 0, 0, 0, 0, 0, 0, 0, 0, 907, 2159, 633, 290, 5, 497, 609, 1030, 0, 0, 0, 0, 0, 0, 0, 0, 24, 67, 37, 155, 553, 1584, 114, 1056, 3204, 1042, 0, 0, 0, 0, 0, 0, 708, 971, 91, 596, 454, 164, 489, 50, 912, 220, 0, 0, 0, 0, 0, 0, 43, 553, 310, 778, 26, 705, 849, 1510, 0, 0, 0, 0, 0, 0, 0, 0, 202, 872, 13, 873, 247, 1412, 3562, 841, 771, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5407, 4878, 6679, 6130, 7836, 4655, 4774, 8793] avg=6144.0000 max=8793 max_rank=7 min=4655 min_rank=5 imbalance_ratio=1.431152 max_load_reduction=40359 max_load_reduction_pct=0.821106 moved_to_echo=26058 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=23 rerouted_tokens_per_expert_current_ep_rank=[272, 15, 204, 1029, 1270, 184, 178, 247, 0, 0, 0, 0, 0, 0, 0, 0, 157, 688, 141, 1660, 902, 175, 61, 415, 416, 395, 0, 0, 0, 0, 0, 0, 1208, 367, 154, 287, 420, 42, 37, 1528, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 1875, 31, 496, 45, 603, 363, 2808, 0, 0, 0, 0, 0, 0, 0, 0, 77, 39, 56, 1401, 56, 517, 211, 674, 526, 724, 0, 0, 0, 0, 0, 0, 1001, 1060, 305, 230, 1907, 209, 2628, 1570, 633, 268, 0, 0, 0, 0, 0, 0, 269, 725, 78, 2404, 9, 2736, 1875, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 269, 162, 41, 309, 585, 662, 333, 2679, 936, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[3399, 5010, 4043, 7487, 4281, 9811, 9145, 5976] avg=6144.0000 max=9811 max_rank=5 min=3399 min_rank=0 imbalance_ratio=1.596842 max_load_reduction=39341 max_load_reduction_pct=0.800395 moved_to_echo=29213 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1276, 236, 734, 969, 1087, 75, 174, 744, 0, 0, 0, 0, 0, 0, 0, 0, 213, 647, 83, 1438, 1676, 2098, 20, 646, 356, 140, 0, 0, 0, 0, 0, 0, 63, 1500, 390, 463, 890, 56, 159, 1292, 0, 0, 0, 0, 0, 0, 0, 0, 1150, 3088, 10, 1954, 56, 215, 136, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 24, 411, 144, 688, 979, 638, 55, 50, 1331, 498, 0, 0, 0, 0, 0, 0, 715, 111, 564, 1033, 1164, 740, 1867, 962, 436, 95, 0, 0, 0, 0, 0, 0, 107, 213, 633, 676, 750, 2548, 774, 721, 0, 0, 0, 0, 0, 0, 0, 0, 133, 83, 39, 765, 48, 738, 2150, 846, 333, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5295, 7317, 4813, 7665, 4818, 7687, 6422, 5135] avg=6144.0000 max=7687 max_rank=5 min=4813 min_rank=2 imbalance_ratio=1.251139 max_load_reduction=41465 max_load_reduction_pct=0.843608 moved_to_echo=24062 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=23 rerouted_tokens_per_expert_current_ep_rank=[1632, 42, 863, 1089, 638, 36, 78, 346, 0, 0, 0, 0, 0, 0, 0, 0, 22, 573, 72, 1043, 2324, 2092, 169, 193, 209, 462, 0, 0, 0, 0, 0, 0, 1024, 1632, 133, 863, 2818, 226, 4, 254, 0, 0, 0, 0, 0, 0, 0, 0, 1572, 874, 25, 252, 13, 704, 495, 398, 0, 0, 0, 0, 0, 0, 0, 0, 982, 35, 56, 1260, 228, 735, 896, 264, 455, 866, 0, 0, 0, 0, 0, 0, 92, 519, 1024, 371, 812, 225, 1383, 620, 758, 313, 0, 0, 0, 0, 0, 0, 80, 772, 63, 275, 192, 1009, 1463, 1255, 0, 0, 0, 0, 0, 0, 0, 0, 200, 564, 10, 3898, 534, 551, 1211, 916, 1095, 0, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4724, 7159, 6954, 4333, 5777, 6117, 5109, 8979] avg=6144.0000 max=8979 max_rank=7 min=4333 min_rank=3 imbalance_ratio=1.461426 max_load_reduction=40173 max_load_reduction_pct=0.817322 moved_to_echo=25982 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=4 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3968, 3200, 6656, 4992, 8960, 2432, 8960, 4480, 4608, 1536, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2304, 4480, 1408, 7168, 2432, 6144, 12800, 7808, 5376, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[896, 6144, 1024, 8064, 11648, 11264, 1280, 4224, 2944, 2304, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7168, 11264, 3328, 6912, 6272, 1280, 1664, 11904, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[9600, 1920, 9984, 9344, 8832, 1792, 1408, 6784, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[2816, 2944, 3840, 6400, 2176, 8448, 3072, 3584, 11392, 5248, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[896, 4096, 1408, 11264, 2048, 13056, 9344, 7552, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7936, 12416, 1920, 5632, 768, 3456, 5376, 12032, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=4 ep_rank=4 layer=24 tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 674, 324, 1114, 1748, 203, 120, 652, 349, 370, 403, 561, 102, 1672, 156, 2030, 315, 611, 366, 277, 353, 576, 1143, 2322, 2637, 0, 290, 21, 2124, 3896, 32, 1826, 209, 2174, 188, 25, 768, 198, 1301, 999, 474, 706, 938, 877, 65, 2583, 1395, 1248, 95, 93, 89, 749, 846, 87, 85, 1470, 236] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=2 ep_rank=2 layer=24 tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 650, 1497, 189, 681, 1133, 716, 473, 859, 6, 2289, 3452, 400, 1781, 91, 1194, 3, 140, 883, 593, 476, 2568, 1906, 2195, 733, 84, 244, 497, 926, 2067, 11, 316, 2632, 315, 359, 40, 112, 545, 948, 2805, 236, 526, 289, 119, 149, 415, 178, 473, 1739, 462, 165, 476, 450, 505, 29, 404, 344] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=1 ep_rank=1 layer=24 tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 519, 151, 1639, 1378, 1391, 3106, 1526, 654, 1875, 2030, 474, 492, 2193, 1038, 276, 805, 169, 923, 28, 343, 3, 840, 1564, 173, 117, 614, 996, 127, 1064, 26, 730, 3001, 224, 1742, 4, 921, 1812, 1160, 17, 48, 54, 2380, 981, 161, 81, 491, 76, 4, 275, 1412, 113, 215, 2549, 19, 44, 982] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=6 ep_rank=6 layer=24 tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 2282, 39, 1388, 900, 110, 23, 1266, 425, 450, 134, 1002, 504, 415, 575, 373, 6, 150, 1468, 961, 431, 8, 473, 2733, 1973, 10, 205, 2, 393, 4757, 157, 14, 1056, 893, 1555, 40, 1502, 488, 3191, 2629, 1912, 1212, 53, 1449, 261, 217, 953, 938, 24, 147, 144, 365, 1739, 735, 17, 238, 243] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=0 ep_rank=0 layer=24 tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 101, 74, 217, 1570, 116, 313, 382, 907, 3573, 3364, 364, 3577, 810, 712, 1259, 267, 669, 415, 113, 287, 83, 32, 1501, 3316, 24, 348, 329, 136, 2197, 178, 128, 1280, 70, 436, 58, 620, 395, 3301, 622, 15, 1722, 441, 467, 318, 192, 1636, 1421, 655, 442, 7, 85, 811, 964, 122, 28, 139] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=3 ep_rank=3 layer=24 tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 1069, 1226, 207, 1431, 723, 601, 121, 1045, 937, 1594, 766, 192, 164, 2698, 782, 249, 28, 2079, 521, 362, 1006, 1274, 1620, 891, 59, 856, 60, 969, 2633, 66, 286, 1888, 2486, 895, 24, 1679, 63, 2350, 1347, 103, 186, 1026, 1249, 278, 150, 395, 1048, 168, 14, 377, 560, 506, 1186, 0, 372, 325] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=7 ep_rank=7 layer=24 tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1634, 299, 2771, 1851, 535, 668, 297, 405, 405, 1972, 436, 245, 394, 1508, 254, 326, 274, 1916, 625, 404, 117, 1073, 1262, 1659, 65, 79, 182, 2114, 3291, 10, 526, 2469, 1181, 900, 19, 268, 295, 3519, 1775, 63, 574, 1627, 772, 439, 614, 162, 506, 15, 109, 6, 289, 801, 455, 11, 112, 1891] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][pre_echo] global_rank=5 ep_rank=5 layer=24 tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 652, 386, 357, 2044, 991, 2488, 145, 1613, 255, 1647, 186, 2187, 724, 716, 1385, 11, 1095, 744, 602, 591, 2446, 736, 3458, 2858, 51, 654, 242, 218, 2418, 244, 126, 823, 1590, 194, 4, 38, 90, 2370, 270, 15, 907, 993, 350, 440, 78, 271, 306, 98, 1181, 391, 35, 136, 524, 132, 91, 245] tokens_per_ep_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] avg=49152.0000 max=49152 max_rank=0 min=49152 min_rank=0 imbalance_ratio=1.000000 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=4 ep_rank=4 layer=24 rerouted_tokens_per_expert_current_ep_rank=[59, 102, 870, 15, 2290, 24, 416, 211, 539, 183, 299, 0, 0, 0, 0, 0, 674, 324, 1114, 805, 203, 120, 652, 349, 0, 0, 0, 0, 0, 0, 0, 0, 370, 36, 561, 102, 1672, 156, 2030, 315, 0, 0, 0, 0, 0, 0, 0, 0, 611, 366, 277, 353, 576, 1143, 242, 2637, 0, 0, 0, 0, 0, 0, 0, 0, 0, 290, 21, 2124, 3156, 32, 1826, 209, 0, 0, 0, 0, 0, 0, 0, 0, 2174, 188, 25, 768, 198, 762, 999, 474, 0, 0, 0, 0, 0, 0, 0, 0, 706, 938, 877, 65, 2583, 1395, 1248, 95, 644, 740, 0, 0, 0, 0, 0, 0, 93, 89, 749, 846, 87, 85, 1470, 236, 2080, 184, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5008, 4241, 5242, 6205, 7658, 5588, 9291, 5919] avg=6144.0000 max=9291 max_rank=6 min=4241 min_rank=1 imbalance_ratio=1.512207 max_load_reduction=39861 max_load_reduction_pct=0.810974 moved_to_echo=28456 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=1 ep_rank=1 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1020, 241, 270, 12, 65, 11, 527, 976, 481, 924, 236, 0, 0, 0, 0, 0, 519, 151, 1639, 635, 1391, 3106, 1526, 654, 0, 0, 0, 0, 0, 0, 0, 0, 1875, 177, 474, 492, 2193, 1038, 276, 805, 0, 0, 0, 0, 0, 0, 0, 0, 169, 923, 28, 343, 3, 840, 163, 173, 0, 0, 0, 0, 0, 0, 0, 0, 117, 614, 996, 127, 862, 26, 730, 3001, 0, 0, 0, 0, 0, 0, 0, 0, 224, 1742, 4, 921, 1812, 679, 17, 48, 0, 0, 0, 0, 0, 0, 0, 0, 54, 2380, 981, 161, 81, 491, 76, 4, 507, 202, 0, 0, 0, 0, 0, 0, 275, 1412, 113, 215, 2549, 19, 44, 982, 1401, 929, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[4763, 9621, 7330, 2642, 6473, 5447, 4937, 7939] avg=6144.0000 max=9621 max_rank=1 min=2642 min_rank=3 imbalance_ratio=1.565918 max_load_reduction=39531 max_load_reduction_pct=0.804260 moved_to_echo=24796 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=6 ep_rank=6 layer=24 rerouted_tokens_per_expert_current_ep_rank=[516, 56, 824, 11, 935, 46, 248, 888, 1324, 61, 154, 0, 0, 0, 0, 0, 2282, 39, 1388, 415, 110, 23, 1266, 425, 0, 0, 0, 0, 0, 0, 0, 0, 450, 12, 1002, 504, 415, 575, 373, 6, 0, 0, 0, 0, 0, 0, 0, 0, 150, 1468, 961, 431, 8, 473, 285, 1973, 0, 0, 0, 0, 0, 0, 0, 0, 10, 205, 2, 393, 3854, 157, 14, 1056, 0, 0, 0, 0, 0, 0, 0, 0, 893, 1555, 40, 1502, 488, 1867, 2629, 1912, 0, 0, 0, 0, 0, 0, 0, 0, 1212, 53, 1449, 261, 217, 953, 938, 24, 331, 903, 0, 0, 0, 0, 0, 0, 147, 144, 365, 1739, 735, 17, 238, 243, 2448, 61, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5063, 5948, 3337, 5749, 5691, 10886, 6341, 6137] avg=6144.0000 max=10886 max_rank=5 min=3337 min_rank=2 imbalance_ratio=1.771810 max_load_reduction=38266 max_load_reduction_pct=0.778524 moved_to_echo=29055 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=0 ep_rank=0 layer=24 rerouted_tokens_per_expert_current_ep_rank=[1832, 290, 474, 313, 1911, 249, 232, 242, 1373, 1536, 272, 0, 0, 0, 0, 0, 101, 74, 217, 716, 116, 313, 382, 907, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 283, 364, 3577, 810, 712, 1259, 267, 0, 0, 0, 0, 0, 0, 0, 0, 669, 415, 113, 287, 83, 32, 151, 3316, 0, 0, 0, 0, 0, 0, 0, 0, 24, 348, 329, 136, 1777, 178, 128, 1280, 0, 0, 0, 0, 0, 0, 0, 0, 70, 436, 58, 620, 395, 1928, 622, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1722, 441, 467, 318, 192, 1636, 1421, 655, 582, 420, 0, 0, 0, 0, 0, 0, 442, 7, 85, 811, 964, 122, 28, 139, 1350, 1545, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[8724, 2826, 10845, 5066, 4200, 4144, 7854, 5493] avg=6144.0000 max=10845 max_rank=2 min=2826 min_rank=1 imbalance_ratio=1.765137 max_load_reduction=38307 max_load_reduction_pct=0.779358 moved_to_echo=21691 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=5 ep_rank=5 layer=24 rerouted_tokens_per_expert_current_ep_rank=[147, 1790, 1657, 148, 877, 189, 369, 133, 983, 750, 350, 0, 0, 0, 0, 0, 652, 386, 357, 941, 991, 2488, 145, 1613, 0, 0, 0, 0, 0, 0, 0, 0, 255, 143, 186, 2187, 724, 716, 1385, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1095, 744, 602, 591, 2446, 736, 360, 2858, 0, 0, 0, 0, 0, 0, 0, 0, 51, 654, 242, 218, 1959, 244, 126, 823, 0, 0, 0, 0, 0, 0, 0, 0, 1590, 194, 4, 38, 90, 1387, 270, 15, 0, 0, 0, 0, 0, 0, 0, 0, 907, 993, 350, 440, 78, 271, 306, 98, 753, 459, 0, 0, 0, 0, 0, 0, 1181, 391, 35, 136, 524, 132, 91, 245, 3098, 754, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[7393, 7573, 5607, 9432, 4317, 3588, 4655, 6587] avg=6144.0000 max=9432 max_rank=3 min=3588 min_rank=5 imbalance_ratio=1.535156 max_load_reduction=39720 max_load_reduction_pct=0.808105 moved_to_echo=19147 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=2 ep_rank=2 layer=24 rerouted_tokens_per_expert_current_ep_rank=[171, 181, 741, 1047, 2152, 503, 406, 183, 393, 1042, 116, 0, 0, 0, 0, 0, 650, 1497, 189, 314, 1133, 716, 473, 859, 0, 0, 0, 0, 0, 0, 0, 0, 6, 199, 3452, 400, 1781, 91, 1194, 3, 0, 0, 0, 0, 0, 0, 0, 0, 140, 883, 593, 476, 2568, 1906, 229, 733, 0, 0, 0, 0, 0, 0, 0, 0, 84, 244, 497, 926, 1675, 11, 316, 2632, 0, 0, 0, 0, 0, 0, 0, 0, 315, 359, 40, 112, 545, 555, 2805, 236, 0, 0, 0, 0, 0, 0, 0, 0, 526, 289, 119, 149, 415, 178, 473, 1739, 251, 392, 0, 0, 0, 0, 0, 0, 462, 165, 476, 450, 505, 29, 404, 344, 1966, 1048, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[6935, 5831, 7126, 7528, 6385, 4967, 4531, 5849] avg=6144.0000 max=7528 max_rank=3 min=4531 min_rank=6 imbalance_ratio=1.225260 max_load_reduction=41624 max_load_reduction_pct=0.846842 moved_to_echo=21732 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=7 ep_rank=7 layer=24 rerouted_tokens_per_expert_current_ep_rank=[434, 174, 184, 8, 1538, 49, 136, 160, 1460, 898, 317, 0, 0, 0, 0, 0, 1634, 299, 2771, 852, 535, 668, 297, 405, 0, 0, 0, 0, 0, 0, 0, 0, 405, 171, 436, 245, 394, 1508, 254, 326, 0, 0, 0, 0, 0, 0, 0, 0, 274, 1916, 625, 404, 117, 1073, 132, 1659, 0, 0, 0, 0, 0, 0, 0, 0, 65, 79, 182, 2114, 2666, 10, 526, 2469, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 900, 19, 268, 295, 2059, 1775, 63, 0, 0, 0, 0, 0, 0, 0, 0, 574, 1627, 772, 439, 614, 162, 506, 15, 682, 625, 0, 0, 0, 0, 0, 0, 109, 6, 289, 801, 455, 11, 112, 1891, 1130, 903, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5358, 7461, 3739, 6200, 8111, 6560, 6016, 5707] avg=6144.0000 max=8111 max_rank=4 min=3739 min_rank=2 imbalance_ratio=1.320150 max_load_reduction=41041 max_load_reduction_pct=0.834981 moved_to_echo=26394 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.moe_layer:[MOE_ECHO_DEBUG][post_echo] global_rank=3 ep_rank=3 layer=24 rerouted_tokens_per_expert_current_ep_rank=[322, 230, 511, 19, 1198, 107, 136, 1439, 975, 726, 245, 0, 0, 0, 0, 0, 1069, 1226, 207, 659, 723, 601, 121, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 937, 138, 766, 192, 164, 2698, 782, 249, 0, 0, 0, 0, 0, 0, 0, 0, 28, 2079, 521, 362, 1006, 1274, 169, 891, 0, 0, 0, 0, 0, 0, 0, 0, 59, 856, 60, 969, 2133, 66, 286, 1888, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 895, 24, 1679, 63, 1375, 1347, 103, 0, 0, 0, 0, 0, 0, 0, 0, 186, 1026, 1249, 278, 150, 395, 1048, 168, 527, 500, 0, 0, 0, 0, 0, 0, 14, 377, 560, 506, 1186, 0, 372, 325, 1451, 730, 0, 0, 0, 0, 0, 0] rerouted_tokens_per_ep_rank=[5908, 5651, 5926, 6330, 6317, 7972, 5527, 5521] avg=6144.0000 max=7972 max_rank=5 min=5521 min_rank=7 imbalance_ratio=1.297526 max_load_reduction=41180 max_load_reduction_pct=0.837809 moved_to_echo=25337 active_clone_links=7 active_echo_slots_local=7 active_home_experts_offloading=5 +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=3 group_rank=3 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=4 group_rank=4 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=6 group_rank=6 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=7 group_rank=7 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=1 group_rank=1 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=0 group_rank=0 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=2 group_rank=2 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][setup_metadata] global_rank=5 group_rank=5 num_tokens=8192 router_topk=6 budget=50304 expected_recv=49152 expected_overflow=0 exact_recv_per_rank=[49152, 49152, 49152, 49152, 49152, 49152, 49152, 49152] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=1 group_rank=1 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[7680, 4096, 7936, 5376, 5248, 8064, 4864, 6272, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=5 group_rank=5 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[8960, 6272, 256, 6016, 3968, 10624, 10496, 2944, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=7 group_rank=7 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[2816, 2688, 2688, 5504, 7040, 512, 2816, 4480, 14976, 6272, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=2 group_rank=2 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[7936, 1280, 7296, 7808, 8192, 7552, 7680, 2048, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=0 group_rank=0 budget=50304 expected_recv=49152 actual_valid_recv=49920 observed_loss=-768 tokens_per_expert=[4608, 3072, 5632, 1664, 11008, 1280, 2560, 4352, 7552, 6144, 2048, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=3 group_rank=3 budget=50304 expected_recv=49152 actual_valid_recv=49792 observed_loss=-640 tokens_per_expert=[3200, 8832, 3840, 3328, 6912, 7552, 1792, 14336, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=6 group_rank=6 budget=50304 expected_recv=49152 actual_valid_recv=49536 observed_loss=-384 tokens_per_expert=[5888, 7808, 6272, 2176, 4352, 5504, 6016, 2816, 4352, 4352, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.token_dispatcher:[MOE_DROP_DEBUG][dispatch] global_rank=4 group_rank=4 budget=50304 expected_recv=49152 actual_valid_recv=49664 observed_loss=-512 tokens_per_expert=[512, 3328, 2432, 7040, 18176, 768, 3968, 13440, 0, 0, 0, 0, 0, 0, 0, 0] +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=1 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=5 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=7 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=2 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=0 received_num_tokens=50304 valid_num_tokens=49920 slack_slots=384 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=6 received_num_tokens=50304 valid_num_tokens=49536 slack_slots=768 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=3 received_num_tokens=50304 valid_num_tokens=49792 slack_slots=512 device_initiated_grouped_gemm=True +WARNING:megatron.core.transformer.moe.experts:[MOE_DROP_DEBUG][experts_pre] global_rank=4 received_num_tokens=50304 valid_num_tokens=49664 slack_slots=640 device_initiated_grouped_gemm=True +WARNING:megatron.core.rerun_state_machine:Result validation enabled +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:482: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +/workspace/DeepEP/deep_ep/hybrid_ep_buffer.py:334: UserWarning: The num_dispatched_tokens is deprecated, it will be removed in the future. + warnings.warn("The num_dispatched_tokens is deprecated, it will be removed in the future.") +W0826 17:20:07.918000 350624 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 350721 closing signal SIGTERM +E0826 17:20:07.923000 350624 torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: -9) local_rank: 5 (pid: 350720) of binary: /usr/bin/python +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 270, in launch_agent + raise ChildFailedError( +torch.distributed.elastic.multiprocessing.errors.ChildFailedError: +============================================================ +pretrain_gpt.py FAILED +------------------------------------------------------------ +Failures: + +------------------------------------------------------------ +Root Cause (first observed failure): +[0]: + time : 2026-08-26_17:20:07 + host : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 + rank : 5 (local_rank: 5) + exitcode : -9 (pid: 350720) + error_file: + traceback : Signal 9 (SIGKILL) received by PID 350720 +============================================================ diff --git a/tierb_study_20260826_144644/hf_watcher.log b/tierb_study_20260826_144644/hf_watcher.log index 814a7a4fffe8975beaa4ec0affb5aa8868341fca..eb4ac5baa99f1c49303832c18c20b7a647c0e292 100644 --- a/tierb_study_20260826_144644/hf_watcher.log +++ b/tierb_study_20260826_144644/hf_watcher.log @@ -58,3 +58,8 @@ Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main [2026-08-26T17:19:17+00:00] backup pass + uploaded: /workspace/tierb_study_20260826_144644 -> MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tierb_study_20260826_144644 +https://huggingface.co/MoE-CUDA-Graph-Analysis/tierb-b200-gbs8/tree/main +Already logged in to Hugging Face as: AbhishekGhosh1998 +Watching /workspace/tierb_study_20260826_144644 -> HF every 600s (kill to stop) +[2026-08-26T17:21:49+00:00] backup pass diff --git a/tierb_study_20260826_144644/hf_watcher.pid b/tierb_study_20260826_144644/hf_watcher.pid index 2076cdc43cbc610e957b25b677e17f0b1dc5bab9..b8d9bb016996eeab714950becbb42ad253d8ff0c 100644 --- a/tierb_study_20260826_144644/hf_watcher.pid +++ b/tierb_study_20260826_144644/hf_watcher.pid @@ -1 +1 @@ -325115 +358610 diff --git a/tierb_study_20260826_144644/study.log b/tierb_study_20260826_144644/study.log index b43d4e6e20e5026897fadaf532e0821061452feb..eb1d49062d7f44e93fbd822dbf715e7f9226c85d 100644 --- a/tierb_study_20260826_144644/study.log +++ b/tierb_study_20260826_144644/study.log @@ -1422,3 +1422,15 @@ driver_repo: /workspace/blackwell_run_scripts @ 6553011 [17:19:04] DSV2_LITE_L24 echo: measured required=1.0234375 at C=2.00 -> validating C=1.0234375 -------------------------------------------------------------------------- [17:19:04] CELL DSV2_LITE_L24__echo__matched__seq8192__gbs8__cap1p0234375 (slots=8) +study_root: /workspace/tierb_study_20260826_144644 +gbs: 8 +stages: 7,4,8,9 +nsys: 1 +hf_repo: MoE-CUDA-Graph-Analysis/tierb-b200-gbs8 +hf_remote_root: tierb_study_20260826_144644 +started: 2026-08-26T17:21:48+00:00 +driver_repo: /workspace/blackwell_run_scripts @ db00a1b + +=== STAGE 7 -- Tier B probe and zero-drop capacity selection + +=== STEP echo__tierbprobe__gbs8